Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 deiflaender
|
c9645c3329 |
@@ -32,7 +32,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.1.7</version>
|
||||
<version>1.1.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeRequest {
|
||||
|
||||
private byte[] document;
|
||||
private String ruleSetId;
|
||||
private ManualRedactions manualRedactions;
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeResult {
|
||||
|
||||
private int numberOfPages;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateRequest {
|
||||
|
||||
private byte[] document;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateResponse {
|
||||
|
||||
private byte[] document;
|
||||
}
|
||||
+10
-13
@@ -1,32 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLog {
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long rulesVersion = -1;
|
||||
|
||||
private String ruleSetId;
|
||||
private String filename;
|
||||
|
||||
|
||||
public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId) {
|
||||
public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion) {
|
||||
|
||||
this.redactionLogEntry = redactionLogEntry;
|
||||
this.dictionaryVersion = dictionaryVersion;
|
||||
this.rulesVersion = rulesVersion;
|
||||
this.ruleSetId = ruleSetId;
|
||||
}
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
|
||||
private long rulesVersion = -1;
|
||||
|
||||
private String filename;
|
||||
|
||||
}
|
||||
|
||||
-7
@@ -32,12 +32,5 @@ public class RedactionLogEntry {
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<Comment> comments = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -12,6 +12,6 @@ import lombok.NoArgsConstructor;
|
||||
public class RedactionRequest {
|
||||
|
||||
private byte[] document;
|
||||
private String ruleSetId;
|
||||
private boolean flatRedaction;
|
||||
private ManualRedactions manualRedactions;
|
||||
}
|
||||
|
||||
+7
-20
@@ -1,28 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
String SERVICE_NAME = "redaction-service-v1";
|
||||
|
||||
String RULE_SET_PARAMETER_NAME = "ruleSetId";
|
||||
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
|
||||
|
||||
@PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
|
||||
|
||||
@PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
|
||||
@PostMapping(value = "/redact", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult redact(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
|
||||
@@ -33,10 +23,7 @@ public interface RedactionResource {
|
||||
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/rules/update"+RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId);
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void testRules(@RequestBody String rules);
|
||||
@PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@RequestBody String rules);
|
||||
|
||||
}
|
||||
@@ -10,9 +10,6 @@
|
||||
</parent>
|
||||
|
||||
<artifactId>redaction-service-server-v1</artifactId>
|
||||
<properties>
|
||||
<slf4j.version>1.7.30</slf4j.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
@@ -23,7 +20,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>1.3.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.drools</groupId>
|
||||
@@ -71,11 +68,6 @@
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
<version>${slf4j.version}</version>
|
||||
</dependency>
|
||||
<!-- spring -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
|
||||
+48
-3
@@ -1,8 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
|
||||
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
|
||||
@@ -10,17 +18,54 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
@Import({DefaultWebMvcConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
|
||||
public class Application {
|
||||
|
||||
@Autowired
|
||||
private RulesClient rulesClient;
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
try {
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
RulesResponse rules = rulesClient.getRules();
|
||||
if (StringUtils.isEmpty(rules.getRules())) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-2
@@ -18,8 +18,6 @@ public class Document {
|
||||
|
||||
private List<Page> pages = new ArrayList<>();
|
||||
private List<Paragraph> paragraphs = new ArrayList<>();
|
||||
private List<Header> headers = new ArrayList<>();
|
||||
private List<Footer> footers = new ArrayList<>();
|
||||
private Map<Integer, List<Entity>> entities = new HashMap<>();
|
||||
private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Footer {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Header {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-3
@@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
@@ -17,8 +16,6 @@ public class Page {
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
|
||||
private List<Rectangle2D> imageBounds;
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
private boolean landscape;
|
||||
|
||||
+18
-44
@@ -5,17 +5,15 @@ import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ClassificationService {
|
||||
@@ -30,7 +28,7 @@ public class ClassificationService {
|
||||
|
||||
List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
|
||||
|
||||
log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
|
||||
System.out.println(document.getFontSizeCounter().getCountPerValue());
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
@@ -41,7 +39,6 @@ public class ClassificationService {
|
||||
|
||||
|
||||
public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
|
||||
@@ -50,33 +47,24 @@ public class ClassificationService {
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// TODO Figure out why this happens.
|
||||
return;
|
||||
}
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Header");
|
||||
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1
|
||||
&& (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
|
||||
&& PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@@ -84,34 +72,20 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-2
@@ -1,8 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface DictionaryClient extends DictionaryResource {
|
||||
}
|
||||
+3
-2
@@ -1,8 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface RulesClient extends RulesResource {
|
||||
}
|
||||
+37
-75
@@ -1,9 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
@@ -13,89 +18,53 @@ import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final PdfVisualisationService pdfVisualisationService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final RedactionLogCreatorService redactionLogCreatorService;
|
||||
private final AnnotationHighlightService annotationHighlightService;
|
||||
private final EntityRedactionService entityRedactionService;
|
||||
private final PdfFlattenService pdfFlattenService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final DictionaryService dictionaryService;
|
||||
private final AnnotationService annotationService;
|
||||
|
||||
|
||||
@Override
|
||||
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
log.info("Starting redaction analysis...");
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
|
||||
annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
|
||||
.getManualRedactions());
|
||||
|
||||
log.info("Document structure analysis successful, starting redaction analysis...");
|
||||
|
||||
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions());
|
||||
redactionLogCreatorService.createRedactionLog(classifiedDoc, pdDocument.getNumberOfPages(), analyzeRequest.getManualRedactions(), analyzeRequest
|
||||
.getRuleSetId());
|
||||
|
||||
log.info("Redaction analysis successful...");
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.sectionGrid(classifiedDoc.getSectionGrid())
|
||||
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
|
||||
.getRulesVersion(), analyzeRequest.getRuleSetId()))
|
||||
.numberOfPages(classifiedDoc.getPages().size())
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(annotateRequest.getDocument()))) {
|
||||
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
dictionaryService.updateDictionary(annotateRequest.getRedactionLog().getRuleSetId());
|
||||
annotationService.annotate(pdDocument, annotateRequest.getRedactionLog(), annotateRequest.getSectionGrid());
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
pdDocument.save(byteArrayOutputStream);
|
||||
return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
|
||||
if (redactionRequest.isFlatRedaction()) {
|
||||
PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
|
||||
return convert(flatDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
return convert(pdDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +77,7 @@ public class RedactionController implements RedactionResource {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size(), pdfSegmentationRequest.getRuleSetId());
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -126,7 +95,7 @@ public class RedactionController implements RedactionResource {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size(), redactionRequest.getRuleSetId());
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -163,35 +132,28 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
|
||||
@Override
|
||||
public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId) {
|
||||
public void updateRules(@RequestBody String rules) {
|
||||
|
||||
droolsExecutionService.updateRules(ruleSetId);
|
||||
droolsExecutionService.updateRules(rules);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void testRules(@RequestBody String rules) {
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
|
||||
|
||||
droolsExecutionService.testRules(rules);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages, String ruleSetId) throws IOException {
|
||||
|
||||
return convert(document, numberOfPages, null, null, 0, 0, ruleSetId);
|
||||
return convert(document, numberOfPages, null, null, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages,
|
||||
List<RedactionLogEntry> redactionLogEntities, SectionGrid sectionGrid,
|
||||
long dictionaryVersion, long rulesVersion, String ruleSetId) throws IOException {
|
||||
List<RedactionLogEntry> redactionLogEntities,
|
||||
SectionGrid sectionGrid, long dictionaryVersion, long rulesVersion) throws IOException {
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
document.save(byteArrayOutputStream);
|
||||
return RedactionResult.builder()
|
||||
.document(byteArrayOutputStream.toByteArray())
|
||||
.numberOfPages(numberOfPages)
|
||||
.redactionLog(new RedactionLog(redactionLogEntities, dictionaryVersion, rulesVersion, ruleSetId))
|
||||
.redactionLog(new RedactionLog(redactionLogEntities,dictionaryVersion, rulesVersion))
|
||||
.sectionGrid(sectionGrid)
|
||||
.build();
|
||||
}
|
||||
|
||||
+28
-135
@@ -1,8 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -30,15 +28,10 @@ import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
@@ -50,17 +43,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
@Setter
|
||||
protected PDPage pdpage;
|
||||
|
||||
@Getter
|
||||
private int minCharWidth;
|
||||
|
||||
@Getter
|
||||
private int maxCharWidth;
|
||||
|
||||
@Getter
|
||||
private int minCharHeight;
|
||||
private int maxCharWidths;
|
||||
|
||||
@Getter
|
||||
private int maxCharHeight;
|
||||
@@ -73,18 +57,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private List<Rectangle2D> imageBounds = new ArrayList<>();
|
||||
|
||||
private float path_x;
|
||||
private float path_y;
|
||||
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
|
||||
public PDFLinesTextStripper() throws IOException {
|
||||
|
||||
super();
|
||||
this.addOperator(new SetStrokingColorSpace());
|
||||
this.addOperator(new SetNonStrokingColorSpace());
|
||||
@@ -108,9 +87,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
this.addOperator(new SetLineWidth());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments)
|
||||
throws IOException {
|
||||
|
||||
String operation = operator.getName();
|
||||
|
||||
@@ -131,11 +110,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
// The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
|
||||
if (pos.getY() > path_y) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
|
||||
.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
|
||||
.getX(), path_y)));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
|
||||
}
|
||||
|
||||
path_x = (float) pos.getX();
|
||||
@@ -156,25 +133,19 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
Point2D p2 = transformPosition(x + width, y + height);
|
||||
|
||||
// Horizontal lines
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
|
||||
// Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
}
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -197,59 +168,12 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
case OperatorName.ENDPATH:
|
||||
graphicsPath.clear();
|
||||
break;
|
||||
|
||||
case OperatorName.DRAW_OBJECT:
|
||||
processImageOperation(arguments);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
super.processOperator(operator, arguments);
|
||||
}
|
||||
|
||||
|
||||
protected void processImageOperation(List<COSBase> arguments) {
|
||||
|
||||
try {
|
||||
COSName objectName = (COSName) arguments.get(0);
|
||||
PDXObject xobject = getResources().getXObject(objectName);
|
||||
if (xobject instanceof PDImageXObject) {
|
||||
PDImageXObject pdfImage = (PDImageXObject) xobject;
|
||||
|
||||
Rectangle2D imageBounds = calculateImagePosition(pdfImage);
|
||||
|
||||
Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
|
||||
.getWidth(), (float) imageBounds.getHeight());
|
||||
|
||||
if (rect.getHeight() > 2 && rect.getWidth() > 2) {
|
||||
this.imageBounds.add(rect);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Problem during image extraction: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
|
||||
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
|
||||
Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
|
||||
|
||||
AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
|
||||
imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
|
||||
imageTransform.translate(0, -pdfImage.getHeight());
|
||||
|
||||
AffineTransform pageTransform = new AffineTransform();
|
||||
pageTransform.concatenate(imageTransform);
|
||||
|
||||
return pageTransform.createTransformedShape(imageBounds).getBounds2D();
|
||||
}
|
||||
|
||||
|
||||
private float floatValue(COSBase value) {
|
||||
|
||||
if (value instanceof COSNumber) {
|
||||
return ((COSNumber) value).floatValue();
|
||||
} else {
|
||||
@@ -257,56 +181,38 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Point2D.Float transformPosition(float x, float y) {
|
||||
|
||||
return super.transformedPoint(x, y);
|
||||
}
|
||||
|
||||
|
||||
private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {
|
||||
|
||||
try {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
|
||||
.toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor()
|
||||
.isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
rulings.addAll(path);
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " or " + getGraphicsState().getNonStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " does not support toRGB");
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if (charWidth < minCharWidth) {
|
||||
minCharWidth = charWidth;
|
||||
}
|
||||
if (charWidth > maxCharWidth) {
|
||||
maxCharWidth = charWidth;
|
||||
}
|
||||
|
||||
int charHeight = (int) textPositions.get(i).getHeightDir();
|
||||
if (charHeight < minCharHeight) {
|
||||
minCharHeight = charHeight;
|
||||
}
|
||||
if (charWidth > maxCharHeight) {
|
||||
if(charHeight > maxCharHeight){
|
||||
maxCharHeight = charHeight;
|
||||
}
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0"))) {
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if(charWidth > maxCharWidths){
|
||||
maxCharWidths = charWidth;
|
||||
}
|
||||
|
||||
if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
@@ -314,21 +220,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")) && i <= textPositions.size() - 2) {
|
||||
if (i > 0 && textPositions.get(i).getUnicode().equals(" ") && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i + 1;
|
||||
@@ -336,29 +236,22 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
|
||||
if (!sublist.isEmpty() && sublist.get(sublist.size() - 1).getUnicode().equals(" ")) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText(PDDocument doc) throws IOException {
|
||||
|
||||
minCharWidth = Integer.MAX_VALUE;
|
||||
maxCharWidth = 0;
|
||||
minCharHeight = Integer.MAX_VALUE;
|
||||
maxCharHeight = 0;
|
||||
maxCharWidths = 0;
|
||||
maxCharWidths = 0;
|
||||
textPositionSequences.clear();
|
||||
imageBounds = new ArrayList<>();
|
||||
rulings.clear();
|
||||
graphicsPath.clear();
|
||||
path_x = 0.0f;
|
||||
|
||||
+1
-3
@@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
@@ -14,11 +13,10 @@ public class ParsedElements {
|
||||
|
||||
private List<TextPositionSequence> sequences;
|
||||
private List<Ruling> rulings;
|
||||
private List<Rectangle2D> imageBounds;
|
||||
|
||||
private boolean landscape;
|
||||
private boolean rotated;
|
||||
|
||||
private float minCharWidth;
|
||||
private float maxCharWidth;
|
||||
private float maxCharHeight;
|
||||
}
|
||||
|
||||
+11
-21
@@ -1,8 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
@@ -12,9 +9,9 @@ import lombok.Value;
|
||||
@Value
|
||||
public class CellValue {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
TextBlock textBlock;
|
||||
|
||||
private int rowSpanStart;
|
||||
int rowSpanStart;
|
||||
|
||||
|
||||
@Override
|
||||
@@ -22,24 +19,17 @@ public class CellValue {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
while (itty.hasNext()) {
|
||||
TextBlock textBlock = itty.next();
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
if (itty.hasNext()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
|
||||
+3
-17
@@ -1,41 +1,27 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Dictionary {
|
||||
|
||||
public static final String RECOMMENDATION_PREFIX = "recommendation_";
|
||||
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionaryModels;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
private Map<String, DictionaryModel> localAccessMap;
|
||||
|
||||
@Getter
|
||||
private long version;
|
||||
|
||||
|
||||
public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
|
||||
this.dictionaryModels = dictionaryModels;
|
||||
this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
this.version = dictionaryVersion;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type){
|
||||
if(!localAccessMap.containsKey(type)){
|
||||
return 0;
|
||||
}
|
||||
return localAccessMap.get(type).getRank();
|
||||
}
|
||||
|
||||
|
||||
public boolean isRecommendation(String type) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
|
||||
+1
-2
@@ -1,7 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -9,7 +8,7 @@ import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryModel implements Serializable {
|
||||
public class DictionaryModel {
|
||||
|
||||
private String type;
|
||||
private int rank;
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
public class DictionaryRepresentation {
|
||||
|
||||
private String ruleSetId;
|
||||
private long dictionaryVersion = -1;
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
private float[] defaultColor;
|
||||
private float[] requestAddColor;
|
||||
private float[] requestRemoveColor;
|
||||
private float[] notRedactedColor;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
|
||||
|
||||
}
|
||||
+3
-15
@@ -12,7 +12,7 @@ import lombok.EqualsAndHashCode;
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity {
|
||||
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private final String word;
|
||||
private final String type;
|
||||
private boolean redaction;
|
||||
@@ -20,10 +20,7 @@ public class Entity {
|
||||
private String legalBasis;
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer end;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
@@ -33,13 +30,8 @@ public class Entity {
|
||||
@EqualsAndHashCode.Include
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -50,13 +42,10 @@ public class Entity {
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
this.textAfter = textAfter;
|
||||
}
|
||||
|
||||
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -64,7 +53,6 @@ public class Entity {
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-29
@@ -32,7 +32,6 @@ public class SearchableText {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("checkstyle:ModifiedControlVariable")
|
||||
public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
|
||||
List<TextPositionSequence> sequencesSubList) {
|
||||
@@ -67,12 +66,9 @@ public class SearchableText {
|
||||
for (int j = 0; j < searchSpace.get(i).length(); j++) {
|
||||
|
||||
if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1)
|
||||
.length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i)
|
||||
.charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions()
|
||||
.isEmpty()) {
|
||||
.charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
}
|
||||
continue;
|
||||
@@ -84,8 +80,8 @@ public class SearchableText {
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace
|
||||
.get(i)
|
||||
if (searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == '-') {
|
||||
|
||||
if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
|
||||
@@ -104,15 +100,14 @@ public class SearchableText {
|
||||
if (counter == searchString.length()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
|
||||
.length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i)
|
||||
.charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && searchSpace.get(i)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive) != ' ') {
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
@@ -135,21 +130,15 @@ public class SearchableText {
|
||||
}
|
||||
|
||||
return finalMatches;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts,
|
||||
String searchString) {
|
||||
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
String asString = buildString(crossSequenceParts);
|
||||
if (!asString.equalsIgnoreCase(searchString)) {
|
||||
return result;
|
||||
}
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
|
||||
|
||||
String plainId = IdBuilder.buildId(crossSequenceParts);
|
||||
String id = plainId;
|
||||
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
int currentPage = -1;
|
||||
int idDiffentPageSuffix = 1;
|
||||
EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
|
||||
@@ -184,12 +173,6 @@ public class SearchableText {
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return buildString(sequences);
|
||||
}
|
||||
|
||||
|
||||
public String buildString(List<TextPositionSequence> sequences) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
|
||||
+102
-171
@@ -2,19 +2,17 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
import lombok.Builder;
|
||||
@@ -47,28 +45,18 @@ public class Section {
|
||||
|
||||
private Map<String, CellValue> tabularData;
|
||||
|
||||
private Dictionary dictionary;
|
||||
|
||||
private SearchableText searchableText;
|
||||
|
||||
|
||||
public boolean rowEquals(String headerName, String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
|
||||
.toString()
|
||||
.getTextBlock()
|
||||
.getText()
|
||||
.equals(value);
|
||||
}
|
||||
|
||||
|
||||
public boolean hasTableHeader(String headerName) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName);
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesType(String type) {
|
||||
|
||||
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
|
||||
@@ -81,40 +69,12 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Set<Entity> expanded = new HashSet<>();
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
|
||||
.getRedactionReason(), entity.getLegalBasis()));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
|
||||
|
||||
public void redact(String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
boolean hasRecommendactionDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
if (entity.getType().equals(type) || hasRecommendactionDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
@@ -127,10 +87,10 @@ public class Section {
|
||||
|
||||
public void redactNot(String type, int ruleNumber, String reason) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
boolean hasRecommendactionDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
if (entity.getType().equals(type) || hasRecommendactionDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(false);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
@@ -155,8 +115,8 @@ public class Section {
|
||||
|
||||
public void addHintAnnotation(String value, String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true);
|
||||
entities.addAll(found);
|
||||
}
|
||||
|
||||
|
||||
@@ -168,88 +128,24 @@ public class Section {
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void recommendLineAfter(String start, String asType) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
|
||||
String trimmedValue = value.trim();
|
||||
String cleanValue;
|
||||
if (trimmedValue.startsWith(":")) {
|
||||
cleanValue = trimmedValue.substring(1).trim();
|
||||
} else {
|
||||
cleanValue = trimmedValue;
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>())
|
||||
.add(cleanValue);
|
||||
}
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
public void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
|
||||
String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addRecommendationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -261,16 +157,26 @@ public class Section {
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -283,16 +189,12 @@ public class Section {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String[] lines = value.split("\n");
|
||||
|
||||
for (String line : lines) {
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false);
|
||||
|
||||
if (line.trim().length() <= 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
@@ -300,6 +202,61 @@ public class Section {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseinsensitive) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
String text = caseinsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseinsensitive ? value.toLowerCase() : value;
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = text.indexOf(searchValue, stopIndex);
|
||||
stopIndex = startIndex + searchValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(text.charAt(startIndex - 1)) || isSeparator(text
|
||||
.charAt(startIndex - 1))) && (stopIndex == text.length() || isSeparator(text.charAt(stopIndex)))) {
|
||||
found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
|
||||
return removeEntitiesContainedInLarger(found);
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
@@ -323,27 +280,6 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseInsensitive ? value.toLowerCase() : value;
|
||||
|
||||
Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true);
|
||||
|
||||
found.forEach(entity -> {
|
||||
if (redacted) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
|
||||
boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
|
||||
@@ -355,24 +291,19 @@ public class Section {
|
||||
} else {
|
||||
String word = value.toString();
|
||||
|
||||
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
|
||||
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber);
|
||||
entity.setRedaction(redact);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setTargetSequences(value.getTextBlocks()
|
||||
.stream()
|
||||
.map(TextBlock::getSequences)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
|
||||
entity.setTargetSequences(value.getTextBlock()
|
||||
.getSequences()); // Make sure no other cells with same content are highlighted
|
||||
entity.setLegalBasis(legalBasis);
|
||||
|
||||
Set<Entity> singleEntitySet = new HashSet<>();
|
||||
singleEntitySet.add(entity);
|
||||
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.remove(entity);
|
||||
entities.add(entity);
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
|
||||
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
entities = removeEntitiesContainedInLarger(entities);
|
||||
|
||||
if (addAsRecommendations && !isLocal()) {
|
||||
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
|
||||
|
||||
-271
@@ -1,271 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
|
||||
log.info("Annotating document.");
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
|
||||
List<SectionRectangle> sectionRectangles = sectionGrid.getRectanglesPerPage().get(page);
|
||||
if (sectionRectangles != null && !sectionRectangles.isEmpty()) {
|
||||
drawSectionGrid(document, pdPage, sectionRectangles);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
|
||||
if (logEntries != null && !logEntries.isEmpty()) {
|
||||
addAnnotations(logEntries, pdPage, page);
|
||||
}
|
||||
}
|
||||
log.info("Finished document annotation.");
|
||||
}
|
||||
|
||||
|
||||
private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
|
||||
for (RedactionLogEntry entry : logEntries) {
|
||||
annotations.addAll(createAnnotation(entry, page, pdPage.getMediaBox(), pdPage.getCropBox()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, PDRectangle mediaBox,
|
||||
PDRectangle cropBox) {
|
||||
|
||||
List<PDAnnotation> annotations = new ArrayList<>();
|
||||
|
||||
List<Rectangle> rectangles = redactionLogEntry.getPositions()
|
||||
.stream()
|
||||
.filter(pos -> pos.getPage() == page)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (rectangles.isEmpty()) {
|
||||
return annotations;
|
||||
}
|
||||
|
||||
PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
|
||||
annotation.constructAppearances();
|
||||
PDRectangle pdRectangle = toPDRectangle(rectangles, mediaBox, cropBox);
|
||||
annotation.setRectangle(pdRectangle);
|
||||
annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
|
||||
if (!redactionLogEntry.isHint()) {
|
||||
annotation.setContents(createAnnotationContent(redactionLogEntry));
|
||||
}
|
||||
annotation.setTitlePopup(redactionLogEntry.getId());
|
||||
annotation.setAnnotationName(redactionLogEntry.getId());
|
||||
annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
|
||||
annotations.add(annotation);
|
||||
|
||||
if (redactionLogEntry.getComments() != null) {
|
||||
for (Comment comment : redactionLogEntry.getComments()) {
|
||||
PDAnnotationText txtAnnot = new PDAnnotationText();
|
||||
txtAnnot.setAnnotationName(comment.getId());
|
||||
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
|
||||
txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
|
||||
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));
|
||||
txtAnnot.setTitlePopup(comment.getUser());
|
||||
txtAnnot.setContents(comment.getText());
|
||||
txtAnnot.setRectangle(pdRectangle);
|
||||
annotations.add(txtAnnot);
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
|
||||
private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.isManual()) {
|
||||
return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
return "\nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
|
||||
.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
|
||||
|
||||
private PDRectangle toPDRectangle(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float lowerLeftX = Float.MAX_VALUE;
|
||||
float upperRightX = 0;
|
||||
float lowerLeftY = 0;
|
||||
float upperRightY = Float.MAX_VALUE;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
if (rectangle.getTopLeft().getX() < lowerLeftX) {
|
||||
lowerLeftX = rectangle.getTopLeft().getX();
|
||||
}
|
||||
if (rectangle.getTopLeft().getX() + rectangle.getWidth() > upperRightX) {
|
||||
upperRightX = rectangle.getTopLeft().getX() + rectangle.getWidth();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() + rectangle.getHeight() > lowerLeftY) {
|
||||
lowerLeftY = rectangle.getTopLeft().getY() + rectangle.getHeight();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() < upperRightY) {
|
||||
upperRightY = rectangle.getTopLeft().getY();
|
||||
}
|
||||
}
|
||||
|
||||
var x1 = lowerLeftX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = lowerLeftY + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = upperRightX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = upperRightY - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
PDRectangle annotationPosition = new PDRectangle();
|
||||
annotationPosition.setLowerLeftX(x1);
|
||||
annotationPosition.setLowerLeftY(y1);
|
||||
annotationPosition.setUpperRightX(x2);
|
||||
annotationPosition.setUpperRightY(y2);
|
||||
return annotationPosition;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoints(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float[] quadPoints = new float[rectangles.size() * 8];
|
||||
int i = 0;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
float[] quadPoint = toQuadPoint(rectangle, mediaBox, cropBox);
|
||||
for (int j = 0; j <= 7; j++) {
|
||||
quadPoints[i + j] = quadPoint[j];
|
||||
}
|
||||
i += 8;
|
||||
}
|
||||
return quadPoints;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoint(Rectangle rectangle, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
var x1 = rectangle.getTopLeft().getX() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = rectangle.getTopLeft().getY() + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = rectangle.getTopLeft()
|
||||
.getX() + rectangle.getWidth() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = rectangle.getTopLeft()
|
||||
.getY() + rectangle.getHeight() - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
|
||||
// of the area to be highlighted
|
||||
return new float[]{x1, y1, x2, y2, x1, y2 - rectangle.getHeight(), x2, y1 - rectangle.getHeight()};
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionGrid(PDDocument document, PDPage pdPage,
|
||||
List<SectionRectangle> sectionRectangles) throws IOException {
|
||||
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
for (SectionRectangle sectionRectangle : sectionRectangles) {
|
||||
drawSectionRectangle(contentStream, sectionRectangle);
|
||||
drawSectionPartNumberText(contentStream, sectionRectangle);
|
||||
drawTableCells(contentStream, sectionRectangle);
|
||||
}
|
||||
contentStream.close();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionRectangle(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.LIGHT_GRAY);
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.addRect(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY(), sectionRectangle.getWidth(), sectionRectangle.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionPartNumberText(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.beginText();
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
if (sectionRectangle.getTableCells() == null) {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY() + sectionRectangle.getHeight());
|
||||
} else {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY());
|
||||
}
|
||||
contentStream.showText(sectionRectangle.getPart() + "/" + sectionRectangle.getNumberOfParts());
|
||||
contentStream.endText();
|
||||
}
|
||||
|
||||
|
||||
private void drawTableCells(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
if (sectionRectangle.getTableCells() != null) {
|
||||
for (CellRectangle cell : sectionRectangle.getTableCells()) {
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.setStrokingColor(Color.CYAN);
|
||||
contentStream.addRect(cell.getTopLeft().getX(), cell.getTopLeft()
|
||||
.getY(), cell.getWidth(), cell.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Map<Integer, List<RedactionLogEntry>> convertRedactionLog(RedactionLog redactionLog) {
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = new HashMap<>();
|
||||
if (redactionLog == null) {
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
int page = 0;
|
||||
for (Rectangle position : entry.getPositions()) {
|
||||
if (position.getPage() != page) {
|
||||
redactionLogPerPage.computeIfAbsent(position.getPage(), x -> new ArrayList<>()).add(entry);
|
||||
page = position.getPage();
|
||||
}
|
||||
}
|
||||
}
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
|
||||
}
|
||||
+66
-75
@@ -1,19 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
|
||||
import feign.FeignException;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
@@ -24,6 +10,21 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -32,51 +33,60 @@ public class DictionaryService {
|
||||
|
||||
private final DictionaryClient dictionaryClient;
|
||||
|
||||
@Getter
|
||||
private long dictionaryVersion = -1;
|
||||
|
||||
private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private float[] defaultColor;
|
||||
|
||||
@Getter
|
||||
private float[] requestAddColor;
|
||||
|
||||
@Getter
|
||||
private float[] requestRemoveColor;
|
||||
|
||||
@Getter
|
||||
private float[] notRedactedColor;
|
||||
|
||||
|
||||
public void updateDictionary(String ruleSetId) {
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
long version = dictionaryClient.getVersion(ruleSetId);
|
||||
|
||||
var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
|
||||
|
||||
if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(ruleSetId, version);
|
||||
public void updateDictionary() {
|
||||
|
||||
long version = dictionaryClient.getVersion();
|
||||
if (version > dictionaryVersion) {
|
||||
dictionaryVersion = version;
|
||||
updateDictionaryEntry();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void updateDictionaryEntry(String ruleSetId, long version) {
|
||||
private void updateDictionaryEntry() {
|
||||
|
||||
try {
|
||||
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
|
||||
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId);
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes();
|
||||
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
|
||||
|
||||
List<DictionaryModel> dictionary = typeResponse.getTypes()
|
||||
dictionary = typeResponse.getTypes()
|
||||
.stream()
|
||||
.map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
|
||||
.isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
localAccessMap.clear();
|
||||
dictionary.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
|
||||
dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
|
||||
|
||||
Colors colors = dictionaryClient.getColors(ruleSetId);
|
||||
|
||||
dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
|
||||
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
|
||||
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
|
||||
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
|
||||
dictionaryRepresentation.setRuleSetId(ruleSetId);
|
||||
dictionaryRepresentation.setDictionaryVersion(version);
|
||||
dictionaryRepresentation.setDictionary(dictionary);
|
||||
|
||||
dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
|
||||
Colors colors = dictionaryClient.getColors();
|
||||
defaultColor = convertColor(colors.getDefaultColor());
|
||||
requestAddColor = convertColor(colors.getRequestAdd());
|
||||
requestRemoveColor = convertColor(colors.getRequestRemove());
|
||||
notRedactedColor = convertColor(colors.getNotRedacted());
|
||||
}
|
||||
} catch (FeignException e) {
|
||||
log.warn("Got some unknown feignException", e);
|
||||
@@ -85,12 +95,12 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
|
||||
public void updateExternalDictionary(Dictionary dictionary){
|
||||
dictionary.getDictionaryModels().forEach(dm -> {
|
||||
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
|
||||
dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
|
||||
long externalVersion = dictionaryClient.getVersion(ruleSetId);
|
||||
if (externalVersion == dictionary.getVersion() + 1) {
|
||||
if(dm.isRecommendation() && !dm.getLocalEntries().isEmpty()){
|
||||
dictionaryClient.addEntries(dm.getType(), new ArrayList<>(dm.getLocalEntries()), false);
|
||||
long externalVersion = dictionaryClient.getVersion();
|
||||
if(externalVersion == dictionary.getVersion() + 1){
|
||||
dictionary.setVersion(externalVersion);
|
||||
}
|
||||
}
|
||||
@@ -101,13 +111,13 @@ public class DictionaryService {
|
||||
private Set<String> convertEntries(TypeResult t) {
|
||||
|
||||
if (t.isCaseInsensitive()) {
|
||||
return dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
|
||||
return dictionaryClient.getDictionaryForType(t.getType())
|
||||
.getEntries()
|
||||
.stream()
|
||||
.map(String::toLowerCase)
|
||||
.collect(Collectors.toSet());
|
||||
} else {
|
||||
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId()).getEntries());
|
||||
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType()).getEntries());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,9 +129,9 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) {
|
||||
public boolean isCaseInsensitiveDictionary(String type) {
|
||||
|
||||
DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type);
|
||||
if (dictionaryModel != null) {
|
||||
return dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
@@ -129,28 +139,28 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public float[] getColor(String type, String ruleSetId) {
|
||||
public float[] getColor(String type) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.getColor();
|
||||
}
|
||||
return dictionariesByRuleSets.get(ruleSetId).getDefaultColor();
|
||||
return defaultColor;
|
||||
}
|
||||
|
||||
|
||||
public boolean isHint(String type, String ruleSetId) {
|
||||
public boolean isHint(String type) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isHint();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isRecommendation(String type, String ruleSetId) {
|
||||
public boolean isRecommendation(String type) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isRecommendation();
|
||||
}
|
||||
@@ -158,27 +168,8 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public Dictionary getDeepCopyDictionary(String ruleSetId) {
|
||||
List<DictionaryModel> copy = new ArrayList<>();
|
||||
|
||||
var representation = dictionariesByRuleSets.get(ruleSetId);
|
||||
var dictionary = dictionariesByRuleSets.get(ruleSetId).getDictionary();
|
||||
dictionary.forEach(dm -> {
|
||||
copy.add(SerializationUtils.clone(dm));
|
||||
});
|
||||
|
||||
return new Dictionary(copy, representation.getDictionaryVersion());
|
||||
public Dictionary getThreadSafeDictionary(){
|
||||
return new Dictionary(new ArrayList<>(dictionary), new HashMap<>(localAccessMap), dictionaryVersion);
|
||||
}
|
||||
|
||||
public float[] getRequestRemoveColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
|
||||
}
|
||||
|
||||
public float[] getNotRedactedColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
|
||||
}
|
||||
|
||||
public float[] getRequestAddColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
|
||||
}
|
||||
}
|
||||
+21
-79
@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
@@ -13,13 +11,14 @@ import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@@ -28,23 +27,13 @@ public class DroolsExecutionService {
|
||||
|
||||
private final RulesClient rulesClient;
|
||||
|
||||
private Map<String, KieContainer> kieContainers = new HashMap<>();
|
||||
@Autowired
|
||||
private KieContainer kieContainer;
|
||||
|
||||
private Map<String, Long> rulesVersionPerRuleSetId = new HashMap<>();
|
||||
@Getter
|
||||
private long rulesVersion = -1;
|
||||
|
||||
|
||||
public KieContainer getKieContainer(String ruleSetId) {
|
||||
|
||||
KieContainer container = kieContainers.get(ruleSetId);
|
||||
if (container == null) {
|
||||
return createOrUpdateKieContainer(ruleSetId);
|
||||
} else {
|
||||
return container;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Section executeRules(KieContainer kieContainer, Section section) {
|
||||
public Section executeRules(Section section) {
|
||||
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("section", section);
|
||||
@@ -56,81 +45,34 @@ public class DroolsExecutionService {
|
||||
|
||||
}
|
||||
|
||||
public void updateRules() {
|
||||
|
||||
public KieContainer updateRules(String ruleSetId) {
|
||||
|
||||
long version = rulesClient.getVersion(ruleSetId);
|
||||
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
|
||||
if (rulesVersion == null) {
|
||||
rulesVersion = -1L;
|
||||
long version = rulesClient.getVersion();
|
||||
if (version > rulesVersion) {
|
||||
rulesVersion = version;
|
||||
updateRules(rulesClient.getRules().getRules());
|
||||
}
|
||||
|
||||
if (version > rulesVersion.longValue()) {
|
||||
rulesVersionPerRuleSetId.put(ruleSetId, version);
|
||||
return createOrUpdateKieContainer(ruleSetId);
|
||||
}
|
||||
return getKieContainer(ruleSetId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieContainer createOrUpdateKieContainer(String ruleSetId) {
|
||||
public void updateRules(String drlAsString) {
|
||||
|
||||
try {
|
||||
|
||||
RulesResponse rules = rulesClient.getRules(ruleSetId);
|
||||
if (rules == null || StringUtils.isEmpty(rules.getRules())) {
|
||||
if (StringUtils.isEmpty(drlAsString)) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices);
|
||||
|
||||
var container = kieContainers.get(ruleSetId);
|
||||
if (container != null) {
|
||||
container.updateToVersion(kieModule.getReleaseId());
|
||||
return container;
|
||||
}
|
||||
|
||||
container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
kieContainers.put(ruleSetId, container);
|
||||
return container;
|
||||
InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
kieContainer.updateToVersion(kieModule.getReleaseId());
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) {
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
return kieBuilder.getKieModule();
|
||||
}
|
||||
|
||||
|
||||
public void testRules(String rules) {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule("test-rules", rules, kieServices);
|
||||
var container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
container.newKieSession();
|
||||
container.dispose();
|
||||
}
|
||||
|
||||
|
||||
public long getRulesVersion(String ruleSetId) {
|
||||
|
||||
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
|
||||
if (rulesVersion == null) {
|
||||
return -1;
|
||||
}
|
||||
return rulesVersion.longValue();
|
||||
}
|
||||
|
||||
}
|
||||
+140
-184
@@ -6,21 +6,18 @@ import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
@@ -31,34 +28,28 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EntityRedactionService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final SurroundingWordsService surroundingWordsService;
|
||||
|
||||
|
||||
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
|
||||
public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
|
||||
|
||||
log.info("Processing document.");
|
||||
dictionaryService.updateDictionary();
|
||||
droolsExecutionService.updateRules();
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion();
|
||||
|
||||
dictionaryService.updateDictionary(ruleSetId);
|
||||
KieContainer container = droolsExecutionService.updateRules(ruleSetId);
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
|
||||
Dictionary dictionary = dictionaryService.getThreadSafeDictionary();
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null));
|
||||
|
||||
if (dictionary.hasLocalEntries()) {
|
||||
|
||||
@@ -70,9 +61,12 @@ public class EntityRedactionService {
|
||||
}
|
||||
});
|
||||
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
documentEntities.removeAll(foundByLocal);
|
||||
documentEntities.addAll(foundByLocal);
|
||||
|
||||
removeEntitiesContainedInLarger(documentEntities);
|
||||
}
|
||||
|
||||
for (Entity entity : documentEntities) {
|
||||
@@ -87,217 +81,136 @@ public class EntityRedactionService {
|
||||
.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
|
||||
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
|
||||
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
|
||||
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
|
||||
.getLegalBasis()));
|
||||
}
|
||||
}
|
||||
|
||||
dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
|
||||
dictionaryService.updateExternalDictionary(dictionary);
|
||||
|
||||
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
|
||||
classifiedDoc.setRulesVersion(rulesVersion);
|
||||
|
||||
log.info("Finished document processing.");
|
||||
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(Document classifiedDoc, KieContainer kieContainer,
|
||||
ManualRedactions manualRedactions, Dictionary dictionary, boolean local,
|
||||
private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions,
|
||||
Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>();
|
||||
|
||||
AtomicInteger sectionNumber = new AtomicInteger(1);
|
||||
int sectionNumber = 1;
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
SearchableText searchableText = paragraph.getSearchableText();
|
||||
|
||||
List<Table> tables = paragraph.getTables();
|
||||
|
||||
for (Table table : tables) {
|
||||
if (table.getColCount() == 2) {
|
||||
sectionSearchableTextPairs.addAll(processTableAsOneText(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
} else {
|
||||
sectionSearchableTextPairs.addAll(processTablePerRow(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
boolean singleCellTable = table.getRowCount() == 1 && table.getColCount() == 1;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
Map<String, CellValue> tabularData = new HashMap<>();
|
||||
int start = 0;
|
||||
for (Cell cell : row) {
|
||||
if (!singleCellTable && cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber);
|
||||
int cellStart = start;
|
||||
cell.getHeaderCells().forEach(headerCell -> {
|
||||
StringBuilder headerBuilder = new StringBuilder();
|
||||
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
|
||||
String headerName = headerBuilder.toString()
|
||||
.replaceAll("\n", "")
|
||||
.replaceAll(" ", "")
|
||||
.replaceAll("-", "");
|
||||
tabularData.put(headerName, new CellValue(cell.getTextBlocks().get(0), cellStart));
|
||||
});
|
||||
start = start + cell.toString().length() + 1; // include automatically appended white space
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
}
|
||||
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber)
|
||||
.tabularData(tabularData)
|
||||
.build(), searchableRow));
|
||||
|
||||
sectionNumber++;
|
||||
}
|
||||
sectionNumber.incrementAndGet();
|
||||
sectionNumber++;
|
||||
}
|
||||
sectionSearchableTextPairs.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
|
||||
.getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
for (Header header : classifiedDoc.getHeaders()) {
|
||||
sectionSearchableTextPairs.add(processText(header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
|
||||
Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
|
||||
|
||||
for (Footer footer : classifiedDoc.getFooters()) {
|
||||
sectionSearchableTextPairs.add(processText(footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(paragraph.getHeadline())
|
||||
.sectionNumber(sectionNumber)
|
||||
.build(), searchableText));
|
||||
|
||||
sectionNumber++;
|
||||
}
|
||||
|
||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
||||
documentEntities.addAll(analysedRowSection.getEntities());
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(sectionSearchableTextPair.getSection());
|
||||
documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair.getSearchableText(), dictionary));
|
||||
|
||||
analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
|
||||
if (dictionary.isRecommendation(key)) {
|
||||
if (dictionary.isRecommendation(key)){
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
if (!dictionary.containsValue(key, value)) {
|
||||
if (!dictionary.containsValue(key, value)){
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key) == null) {
|
||||
log.warn("Dictionary {} is null", key);
|
||||
}
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
|
||||
log.warn("Dictionary {} localEntries is null", key);
|
||||
}
|
||||
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
return documentEntities;
|
||||
}
|
||||
|
||||
|
||||
private List<SectionSearchableTextPair> processTablePerRow(Table table, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
removeEntitiesContainedInLarger(entities);
|
||||
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
Map<String, CellValue> tabularData = new HashMap<>();
|
||||
int start = 0;
|
||||
List<Integer> cellStarts = new ArrayList<>();
|
||||
for (Cell cell : row) {
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
int cellStart = start;
|
||||
|
||||
if (!cell.isHeaderCell()) {
|
||||
cell.getHeaderCells().forEach(headerCell -> {
|
||||
StringBuilder headerBuilder = new StringBuilder();
|
||||
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
|
||||
String headerName = headerBuilder.toString()
|
||||
.replaceAll("\n", "")
|
||||
.replaceAll(" ", "")
|
||||
.replaceAll("-", "");
|
||||
tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
|
||||
});
|
||||
}
|
||||
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
// TODO avoid cell overlap merging.
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
cellStarts.add(cellStart);
|
||||
start = start + cell.toString().trim().length() + 1;
|
||||
}
|
||||
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.tabularData(tabularData)
|
||||
.searchableText(searchableRow)
|
||||
.dictionary(dictionary)
|
||||
.build(), searchableRow));
|
||||
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
return sectionSearchableTextPairs;
|
||||
}
|
||||
|
||||
|
||||
private List<SectionSearchableTextPair> processTableAsOneText(Table table, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
SearchableText entireTableText = new SearchableText();
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
entireTableText.addAll(textBlock.getSequences());
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
for (Entity entity : entities) {
|
||||
if (entity.getPositionSequences().isEmpty()) {
|
||||
entity.setPositionSequences(text.getSequences(entity.getWord(), dictionary.isCaseInsensitiveDictionary(entity
|
||||
.getType()), entity.getTargetSequences()));
|
||||
}
|
||||
}
|
||||
|
||||
Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(entireTableText.getAsStringWithLinebreaks())
|
||||
.searchText(entireTableText.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(entireTableText)
|
||||
.dictionary(dictionary)
|
||||
.build(), entireTableText));
|
||||
|
||||
return sectionSearchableTextPairs;
|
||||
}
|
||||
|
||||
|
||||
private SectionSearchableTextPair processText(SearchableText searchableText, List<TextBlock> paragraphTextBlocks,
|
||||
String headline, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
|
||||
Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
|
||||
|
||||
return new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(headline)
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(searchableText)
|
||||
.dictionary(dictionary)
|
||||
.build(), searchableText);
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local) {
|
||||
List<DictionaryModel> dictionary, boolean local) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
String searchableString = searchableText.toString();
|
||||
@@ -306,15 +219,58 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
String lowercaseInputString = searchableString.toLowerCase();
|
||||
for (DictionaryModel model : dictionary.getDictionaryModels()) {
|
||||
for (DictionaryModel model : dictionary) {
|
||||
if (model.isCaseInsensitive()) {
|
||||
found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
|
||||
found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
|
||||
} else {
|
||||
found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
|
||||
found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
|
||||
}
|
||||
}
|
||||
removeEntitiesContainedInLarger(found);
|
||||
return found;
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
for (String value : values) {
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(value, stopIndex);
|
||||
stopIndex = startIndex + value.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
|
||||
.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
|
||||
|
||||
-374
@@ -1,374 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.IdRemoval;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Status;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogCreatorService {
|
||||
|
||||
private static final String IMAGE = "image";
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
|
||||
String ruleSetId) {
|
||||
|
||||
log.info("Creating redaction log.");
|
||||
|
||||
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
|
||||
|
||||
for (int page = 1; page <= numberOfPages; page++) {
|
||||
|
||||
addSectionGrid(classifiedDoc, page);
|
||||
|
||||
if (classifiedDoc.getEntities().get(page) != null) {
|
||||
addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
if (manualRedactionPages.contains(page)) {
|
||||
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
|
||||
addImageEntries(classifiedDoc, page, ruleSetId);
|
||||
}
|
||||
}
|
||||
|
||||
log.info("Finished redaction log creation.");
|
||||
}
|
||||
|
||||
|
||||
private void addImageEntries(Document classifiedDoc, int pageNumber, String ruleSetId) {
|
||||
|
||||
for (Rectangle2D imageBounds : classifiedDoc.getPages().get(pageNumber - 1).getImageBounds()) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(IdBuilder.buildId(imageBounds, pageNumber))
|
||||
.color(getColor(IMAGE, ruleSetId))
|
||||
.type(IMAGE)
|
||||
.redacted(false)
|
||||
.isHint(true)
|
||||
.manual(false)
|
||||
.isDictionaryEntry(false)
|
||||
.isRecommendation(false)
|
||||
.positions(List.of(new Rectangle(new Point((float) imageBounds.getX(), (float) imageBounds.getY()), (float) imageBounds
|
||||
.getWidth(), (float) imageBounds.getHeight(), pageNumber)))
|
||||
.build();
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
|
||||
|
||||
Set<Integer> manualRedactionPages = new HashSet<>();
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
manualRedactions.getEntriesToAdd().forEach(entry -> {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
manualRedactionPages.add(pos.getPage());
|
||||
});
|
||||
});
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
|
||||
private void addEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page, String ruleSetId) {
|
||||
|
||||
// Duplicates can exist due table extraction colums over multiple rows.
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
|
||||
entityLoop:
|
||||
for (Entity entity : classifiedDoc.getEntities().get(page)) {
|
||||
|
||||
List<Comment> comments = null;
|
||||
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId);
|
||||
if (processedIds.contains(entityPositionSequence.getId())) {
|
||||
|
||||
// TODO refactor this outer loop jump as soon as we have the time.
|
||||
continue entityLoop;
|
||||
} else {
|
||||
processedIds.add(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
|
||||
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
|
||||
if (manualRemoval.getId().equals(entityPositionSequence.getId())) {
|
||||
comments = manualRedactions.getComments().get(manualRemoval.getId());
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
|
||||
entity.setRedaction(false);
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
|
||||
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
|
||||
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
|
||||
.stream()
|
||||
.flatMap(seq -> seq.getTextPositions().stream())
|
||||
.collect(Collectors.toList()), page);
|
||||
|
||||
if (manualRedactions != null) {
|
||||
comments = manualRedactions.getComments().get(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(comments);
|
||||
redactionLogEntry.getPositions().addAll(rectanglesPerLine);
|
||||
|
||||
}
|
||||
|
||||
redactionLogEntry.setId(entityPositionSequence.getId());
|
||||
|
||||
// FIXME ids should never be null. Figure out why this happens.
|
||||
if (redactionLogEntry.getId() != null) {
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> getRectanglesPerLine(List<TextPosition> textPositions, int page) {
|
||||
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
if (textPositions.size() == 1) {
|
||||
rectangles.add(new TextPositionSequence(textPositions, page).getRectangle());
|
||||
} else {
|
||||
float y = textPositions.get(0).getYDirAdj();
|
||||
int startIndex = 0;
|
||||
for (int i = 1; i < textPositions.size(); i++) {
|
||||
float yDirAdj = textPositions.get(i).getYDirAdj();
|
||||
if (yDirAdj != y) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, i), page).getRectangle());
|
||||
y = yDirAdj;
|
||||
startIndex = i;
|
||||
}
|
||||
}
|
||||
if (startIndex != textPositions.size()) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
|
||||
}
|
||||
}
|
||||
|
||||
return rectangles;
|
||||
}
|
||||
|
||||
|
||||
private void addManualEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page,
|
||||
String ruleSetId) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
|
||||
|
||||
String id = manualRedactionEntry.getId();
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId);
|
||||
|
||||
List<Rectangle> rectanglesOnPage = new ArrayList<>();
|
||||
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
|
||||
if (page == rectangle.getPage()) {
|
||||
rectanglesOnPage.add(rectangle);
|
||||
redactionLogEntry.getPositions().add(rectangle);
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(manualRedactions.getComments().get(id));
|
||||
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
|
||||
String ruleSetId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), ruleSetId, manualRedactionEntry.getStatus()))
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.sectionNumber(manualRedactionEntry.getSectionNumber())
|
||||
.manual(true)
|
||||
.status(manualRedactionEntry.getStatus())
|
||||
.manualRedactionType(ManualRedactionType.ADD)
|
||||
.isDictionaryEntry(false)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.color(getColor(entity, ruleSetId, false))
|
||||
.reason(entity.getRedactionReason())
|
||||
.legalBasis(entity.getLegalBasis())
|
||||
.value(entity.getWord())
|
||||
.type(entity.getType())
|
||||
.redacted(entity.isRedaction())
|
||||
.isHint(isHint(entity, ruleSetId))
|
||||
.isRecommendation(isRecommendation(entity, ruleSetId))
|
||||
.section(entity.getHeadline())
|
||||
.sectionNumber(entity.getSectionNumber())
|
||||
.matchedRule(entity.getMatchedRule())
|
||||
.isDictionaryEntry(entity.isDictionaryEntry())
|
||||
.textAfter(entity.getTextAfter())
|
||||
.textBefore(entity.getTextBefore())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) {
|
||||
|
||||
if (requestedToRemove) {
|
||||
return dictionaryService.getRequestRemoveColor(ruleSetId);
|
||||
}
|
||||
if (!entity.isRedaction() && !isHint(entity, ruleSetId)) {
|
||||
return dictionaryService.getNotRedactedColor(ruleSetId);
|
||||
}
|
||||
return dictionaryService.getColor(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForManualAdd(String type, String ruleSetId, Status status) {
|
||||
|
||||
if (status.equals(Status.REQUESTED)) {
|
||||
return dictionaryService.getRequestAddColor(ruleSetId);
|
||||
} else if (status.equals(Status.DECLINED)) {
|
||||
return dictionaryService.getNotRedactedColor(ruleSetId);
|
||||
}
|
||||
return getColor(type, ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String ruleSetId) {
|
||||
|
||||
return dictionaryService.getColor(type, ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isHint(Entity entity, String ruleSetId) {
|
||||
|
||||
return dictionaryService.isHint(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isRecommendation(Entity entity, String ruleSetId) {
|
||||
|
||||
return dictionaryService.isRecommendation(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private void addSectionGrid(Document classifiedDoc, int page) {
|
||||
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
|
||||
AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
|
||||
|
||||
if (textBlock.getPage() != page) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (textBlock instanceof TextBlock) {
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size()));
|
||||
|
||||
} else if (textBlock instanceof Table) {
|
||||
|
||||
List<CellRectangle> cellRectangles = new ArrayList<>();
|
||||
for (List<Cell> row : ((Table) textBlock).getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (cell != null) {
|
||||
cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size(), cellRectangles));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-140
@@ -1,140 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SurroundingWordsService {
|
||||
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
|
||||
List<Integer> cellstarts) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String searchableString = searchableText.toString();
|
||||
|
||||
if (cellstarts != null) {
|
||||
for (int i = 0; i < cellstarts.size(); i++) {
|
||||
|
||||
int startOffset = cellstarts.get(i);
|
||||
int endOffset = -1;
|
||||
|
||||
if (i + 1 < cellstarts.size()) {
|
||||
endOffset = cellstarts.get(i + 1);
|
||||
} else {
|
||||
endOffset = searchableString.length() - 1;
|
||||
}
|
||||
|
||||
String text = searchableString.substring(startOffset, endOffset);
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
|
||||
int entityStartOffset = entity.getStart() - startOffset;
|
||||
int entityEndOffset = entity.getEnd() - startOffset;
|
||||
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
|
||||
|
||||
int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
|
||||
.getSurroundingWordsOffsetWindow();
|
||||
String textBefore = text.substring(offsetBefore, entityStartOffset);
|
||||
if (!textBefore.isBlank()) {
|
||||
String[] wordsBefore = textBefore.split(" ");
|
||||
int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsBefore.length;
|
||||
if (wordsBefore.length > 0) {
|
||||
entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
|
||||
}
|
||||
}
|
||||
|
||||
int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
|
||||
.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
|
||||
String textAfter = text.substring(entityEndOffset, endOffset);
|
||||
if (!textAfter.isBlank()) {
|
||||
String[] wordsAfter = textAfter.split(" ");
|
||||
int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsAfter.length;
|
||||
if (wordsAfter.length > 0) {
|
||||
entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsBefore(String[] words, int number, boolean endWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int startNumber = words.length > number ? words.length - number : 0;
|
||||
|
||||
for (int i = startNumber; i < words.length; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return endWithSpace ? result + " " : result;
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsAfter(String[] words, int number, boolean startWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < number; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return startWithSpace ? " " + result : result;
|
||||
}
|
||||
|
||||
}
|
||||
-130
@@ -1,130 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@UtilityClass
|
||||
public class EntitySearchUtils {
|
||||
|
||||
public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
|
||||
boolean local) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
for (String value : values) {
|
||||
|
||||
String cleanValue = value.trim();
|
||||
|
||||
if (cleanValue.length() <= 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(cleanValue, stopIndex);
|
||||
stopIndex = startIndex + cleanValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
|
||||
|
||||
Map<String, List<Entity>> entitiesByWord = new HashMap<>();
|
||||
|
||||
for (Entity entity : entities) {
|
||||
entitiesByWord.computeIfAbsent(entity.getWord(), (x) -> new ArrayList<>()).add(entity);
|
||||
}
|
||||
|
||||
for (String word : entitiesByWord.keySet()) {
|
||||
|
||||
List<Entity> orderedEntities = entitiesByWord.get(word)
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
Entity firstEntity = orderedEntities.get(0);
|
||||
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity
|
||||
.getType()), firstEntity.getTargetSequences());
|
||||
|
||||
for (int i = 0; i <= orderedEntities.size() - 1; i++) {
|
||||
try {
|
||||
orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
|
||||
} catch (Exception e) {
|
||||
log.warn("Mismatch between EntityPositionSequence and found Entity!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
removeEntitiesContainedInLarger(entities);
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
|
||||
.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Set<Entity> found, Dictionary dictionary) {
|
||||
found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary));
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Entity found, Dictionary dictionary) {
|
||||
|
||||
if(entities.contains(found)){
|
||||
Entity existing = entities.stream().filter(entity -> entity.equals(found)).findFirst().get();
|
||||
if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){
|
||||
entities.remove(found);
|
||||
}
|
||||
}
|
||||
entities.add(found);
|
||||
}
|
||||
|
||||
public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found){
|
||||
// HashSet keeps old value but we want the new.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
-13
@@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
@@ -24,16 +23,4 @@ public class IdBuilder {
|
||||
|
||||
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
public String buildId(Rectangle2D rectangle2D, int page){
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("x").append(rectangle2D.getX()).append("y").append(rectangle2D.getY()).append("h").append(rectangle2D.getHeight()).append("w").append(rectangle2D.getWidth()).append("p").append(page);
|
||||
|
||||
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-18
@@ -1,28 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class Patterns {
|
||||
|
||||
public static Map<String, Pattern> patternCache = new HashMap<>();
|
||||
|
||||
public static Pattern AUTHOR_TABLE_SPITTER = Pattern.compile("((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2}\\.)+|((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2} )+");
|
||||
|
||||
|
||||
public Pattern getCompiledPattern(String pattern, boolean caseInsensitive) {
|
||||
|
||||
String patternKey = pattern + caseInsensitive;
|
||||
if (patternCache.containsKey(patternKey)) {
|
||||
return patternCache.get(patternKey);
|
||||
}
|
||||
Pattern compiledPattern = Pattern.compile(pattern, caseInsensitive ? Pattern.CASE_INSENSITIVE : 0);
|
||||
patternCache.put(patternKey, compiledPattern);
|
||||
return compiledPattern;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ public class TextNormalizationUtilities {
|
||||
* @return Text without line-break hyphenation.
|
||||
*/
|
||||
public static String removeHyphenLineBreaks(String text) {
|
||||
return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
|
||||
return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
|
||||
}
|
||||
|
||||
}
|
||||
+7
-11
@@ -39,8 +39,6 @@ public class PdfSegmentationService {
|
||||
|
||||
public Document parseDocument(PDDocument pdDocument) throws IOException {
|
||||
|
||||
log.info("Parsing document.");
|
||||
|
||||
Document document = new Document();
|
||||
|
||||
List<Page> pages = new ArrayList<>();
|
||||
@@ -50,7 +48,6 @@ public class PdfSegmentationService {
|
||||
stripper.setPageNumber(pageNumber);
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
stripper.setPdpage(pdPage);
|
||||
stripper.getText(pdDocument);
|
||||
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
@@ -59,17 +56,18 @@ public class PdfSegmentationService {
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isRotated = rotation != 0 && rotation != 360;
|
||||
|
||||
|
||||
ParsedElements parsedElements = ParsedElements.builder()
|
||||
.rulings(stripper.getRulings())
|
||||
.sequences(stripper.getTextPositionSequences())
|
||||
.imageBounds(stripper.getImageBounds())
|
||||
.minCharWidth(stripper.getMinCharWidth())
|
||||
.maxCharWidth(stripper.getMaxCharWidth())
|
||||
.maxCharWidth(stripper.getMaxCharWidths())
|
||||
.maxCharHeight(stripper.getMaxCharWidths())
|
||||
.landscape(isLandscape)
|
||||
.rotated(isRotated)
|
||||
.build();
|
||||
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
|
||||
.getMaxCharWidth(), parsedElements.getMaxCharHeight());
|
||||
|
||||
Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
|
||||
.getVertical());
|
||||
@@ -83,22 +81,20 @@ public class PdfSegmentationService {
|
||||
|
||||
page.setPageNumber(pageNumber);
|
||||
increaseDocumentStatistics(page, document);
|
||||
page.setImageBounds(parsedElements.getImageBounds());
|
||||
pages.add(page);
|
||||
}
|
||||
|
||||
document.setPages(pages);
|
||||
|
||||
classificationService.classifyDocument(document);
|
||||
|
||||
sectionsBuilderService.buildSections(document);
|
||||
|
||||
log.info("Finished document parsing.");
|
||||
|
||||
return document;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void increaseDocumentStatistics(Page page, Document document) {
|
||||
|
||||
if (!page.isLandscape()) {
|
||||
|
||||
+7
-38
@@ -10,8 +10,6 @@ import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
@@ -19,51 +17,32 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SectionsBuilderService {
|
||||
|
||||
public void buildSections(Document document) {
|
||||
|
||||
log.info("Building sections.");
|
||||
|
||||
List<AbstractTextContainer> chunkWords = new ArrayList<>();
|
||||
List<Paragraph> chunkBlockList = new ArrayList<>();
|
||||
List<Header> headers = new ArrayList<>();
|
||||
List<Footer> footers = new ArrayList<>();
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
|
||||
String lastHeadline = "";
|
||||
Table previousTable = null;
|
||||
for (Page page : document.getPages()) {
|
||||
List<TextBlock> header = new ArrayList<>();
|
||||
List<TextBlock> footer = new ArrayList<>();
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
|
||||
if (current.getClassification() == null) {
|
||||
if (current.getClassification() == null || current.getClassification()
|
||||
.equals("Header") || current.getClassification().equals("Footer")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
current.setPage(page.getPageNumber());
|
||||
|
||||
if (current.getClassification().equals("Header")) {
|
||||
header.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.getClassification().equals("Footer")) {
|
||||
footer.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
|
||||
.startsWith("H ") || !document.isHeadlines()) {
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if (document.isHeadlines()) {
|
||||
if(document.isHeadlines()) {
|
||||
lastHeadline = current.getText();
|
||||
}
|
||||
chunkBlockList.add(chunkBlock);
|
||||
@@ -81,8 +60,6 @@ public class SectionsBuilderService {
|
||||
chunkWords.add(current);
|
||||
prev = current;
|
||||
}
|
||||
headers.add(new Header(header));
|
||||
footers.add(new Footer(footer));
|
||||
}
|
||||
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
@@ -90,11 +67,6 @@ public class SectionsBuilderService {
|
||||
chunkBlockList.add(chunkBlock);
|
||||
|
||||
document.setParagraphs(chunkBlockList);
|
||||
document.setHeaders(headers);
|
||||
document.setFooters(footers);
|
||||
|
||||
log.info("Finished section building.");
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -203,9 +175,9 @@ public class SectionsBuilderService {
|
||||
|
||||
private boolean hasInvalidHeaderInformation(Table table) {
|
||||
|
||||
return table.getRows()
|
||||
.stream()
|
||||
.flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
return table.getRows().stream()
|
||||
.flatMap(row -> row.stream()
|
||||
.filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
.findAny()
|
||||
.isEmpty();
|
||||
|
||||
@@ -216,9 +188,6 @@ public class SectionsBuilderService {
|
||||
|
||||
for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
|
||||
List<Cell> row = table.getRows().get(i);
|
||||
if (row.size() == 1) {
|
||||
continue;
|
||||
}
|
||||
boolean allNonHeader = true;
|
||||
for (Cell cell : row) {
|
||||
if (cell.isHeaderCell()) {
|
||||
|
||||
+6
-3
@@ -7,9 +7,12 @@ import lombok.Data;
|
||||
@Data
|
||||
@ConfigurationProperties("redaction-service")
|
||||
public class RedactionServiceSettings {
|
||||
|
||||
private int numberOfSurroundingWords = 3;
|
||||
|
||||
private int surroundingWordsOffsetWindow = 100;
|
||||
/**
|
||||
* Tenant used in single tenant mode.
|
||||
*/
|
||||
private String defaultTenant = "iqser-id";
|
||||
|
||||
private int flattenImageDpi = 100;
|
||||
|
||||
}
|
||||
+2
-13
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
@@ -42,13 +41,8 @@ public class Cell extends Rectangle {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
TextPositionSequence previous = null;
|
||||
while (itty.hasNext()) {
|
||||
|
||||
TextBlock textBlock = itty.next();
|
||||
|
||||
for (TextBlock textBlock : textBlocks) {
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
@@ -60,7 +54,6 @@ public class Cell extends Rectangle {
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
@@ -68,8 +61,4 @@ public class Cell extends Rectangle {
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
-14
@@ -59,16 +59,6 @@ public class Table extends AbstractTextContainer {
|
||||
|
||||
if (rows == null) {
|
||||
rows = computeRows();
|
||||
|
||||
// Ignore rows that does not contain any cells and values.
|
||||
List<List<Cell>> rowsToRemove = new ArrayList<>();
|
||||
for (List<Cell> row: rows){
|
||||
if (row.size() == 1 && row.get(0).getTextBlocks().isEmpty()){
|
||||
rowsToRemove.add(row);
|
||||
}
|
||||
}
|
||||
rows.removeAll(rowsToRemove);
|
||||
|
||||
computeHeaders();
|
||||
}
|
||||
|
||||
@@ -110,10 +100,6 @@ public class Table extends AbstractTextContainer {
|
||||
// we move from left to right and top to bottom
|
||||
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
|
||||
List<Cell> rowCells = rows.get(rowIndex);
|
||||
if(rowCells.size() == 1){
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int colIndex = 0; colIndex < rowCells.size(); colIndex++) {
|
||||
Cell cell = rowCells.get(colIndex);
|
||||
List<Cell> cellsToTheLeft = rowCells.subList(0, colIndex);
|
||||
|
||||
+14
-18
@@ -15,18 +15,12 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRuli
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class RulingCleaningService {
|
||||
|
||||
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
|
||||
|
||||
log.info("Getting clean rulings.");
|
||||
|
||||
public CleanRulings getCleanRulings(List<Ruling> rulings, float maxCharWidth, float maxCharHeight){
|
||||
if (!rulings.isEmpty()) {
|
||||
snapPoints(rulings, minCharWidth, maxCharHeight);
|
||||
snapPoints(rulings, maxCharWidth , maxCharHeight);
|
||||
}
|
||||
|
||||
List<Ruling> vrs = new ArrayList<>();
|
||||
@@ -45,10 +39,13 @@ public class RulingCleaningService {
|
||||
}
|
||||
List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);
|
||||
|
||||
return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
|
||||
return CleanRulings
|
||||
.builder()
|
||||
.vertical(verticalRulingLines)
|
||||
.horizontal(horizontalRulingLines)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
|
||||
|
||||
// collect points and keep a Line -> p1,p2 map
|
||||
@@ -125,14 +122,12 @@ public class RulingCleaningService {
|
||||
|
||||
|
||||
private List<Ruling> collapseOrientedRulings(List<Ruling> lines) {
|
||||
|
||||
int COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT = 1;
|
||||
return collapseOrientedRulings(lines, COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT);
|
||||
}
|
||||
|
||||
|
||||
private List<Ruling> collapseOrientedRulings(List<Ruling> lines, int expandAmount) {
|
||||
|
||||
ArrayList<Ruling> rv = new ArrayList<>();
|
||||
lines.sort((a, b) -> {
|
||||
final float diff = a.getPosition() - b.getPosition();
|
||||
@@ -146,24 +141,25 @@ public class RulingCleaningService {
|
||||
final float lastStart = last.getStart();
|
||||
final float lastEnd = last.getEnd();
|
||||
|
||||
final boolean lastFlipped = lastStart > lastEnd;
|
||||
final boolean lastFlipped = lastStart > lastEnd;
|
||||
final boolean nextFlipped = next_line.getStart() > next_line.getEnd();
|
||||
|
||||
boolean differentDirections = nextFlipped != lastFlipped;
|
||||
float nextS = differentDirections ? next_line.getEnd() : next_line.getStart();
|
||||
float nextS = differentDirections ? next_line.getEnd() : next_line.getStart();
|
||||
float nextE = differentDirections ? next_line.getStart() : next_line.getEnd();
|
||||
|
||||
final float newStart = lastFlipped ? Math.max(nextS, lastStart) : Math.min(nextS, lastStart);
|
||||
final float newEnd = lastFlipped ? Math.min(nextE, lastEnd) : Math.max(nextE, lastEnd);
|
||||
final float newEnd = lastFlipped ? Math.min(nextE, lastEnd) : Math.max(nextE, lastEnd);
|
||||
last.setStartEnd(newStart, newEnd);
|
||||
assert !last.oblique();
|
||||
} else if (next_line.length() == 0) {
|
||||
}
|
||||
else if (next_line.length() == 0) {
|
||||
continue;
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
rv.add(next_line);
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-8
@@ -23,16 +23,11 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TableExtractionService {
|
||||
|
||||
public void extractTables(CleanRulings cleanRulings, Page page) {
|
||||
|
||||
log.info("Extracting tables.");
|
||||
|
||||
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
|
||||
|
||||
List<TextBlock> toBeRemoved = new ArrayList<>();
|
||||
@@ -83,9 +78,6 @@ public class TableExtractionService {
|
||||
}
|
||||
|
||||
page.getTextBlocks().removeAll(toBeRemoved);
|
||||
|
||||
log.info("Finished table extraction.");
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Comparator;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Copied and minimal modified from PDFBox.
|
||||
*/
|
||||
public final class QuickSort {
|
||||
|
||||
private QuickSort() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
|
||||
@Override
|
||||
public int compare(Comparable object1, Comparable object2) {
|
||||
|
||||
return object1.compareTo(object2);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Sorts the given list using the given comparator.
|
||||
*
|
||||
* @param <T> type of the objects to be sorted.
|
||||
* @param list list to be sorted
|
||||
* @param cmp comparator used to compare the objects within the list
|
||||
*/
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> cmp) {
|
||||
|
||||
int size = list.size();
|
||||
if (size < 2) {
|
||||
return;
|
||||
}
|
||||
quicksort(list, cmp);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sorts the given list using compareTo as comparator.
|
||||
*
|
||||
* @param <T> type of the objects to be sorted.
|
||||
* @param list list to be sorted
|
||||
*/
|
||||
public static <T extends Comparable> void sort(List<T> list) {
|
||||
|
||||
sort(list, (Comparator<T>) OBJCOMP);
|
||||
}
|
||||
|
||||
|
||||
private static <T> void quicksort(List<T> list, Comparator<? super T> cmp) {
|
||||
|
||||
Deque<Integer> stack = new ArrayDeque<Integer>();
|
||||
stack.push(0);
|
||||
stack.push(list.size());
|
||||
while (!stack.isEmpty()) {
|
||||
int right = stack.pop();
|
||||
int left = stack.pop();
|
||||
if (right - left < 2) {
|
||||
continue;
|
||||
}
|
||||
int p = left + ((right - left) / 2);
|
||||
p = partition(list, cmp, p, left, right);
|
||||
|
||||
stack.push(p + 1);
|
||||
stack.push(right);
|
||||
|
||||
stack.push(left);
|
||||
stack.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static <T> int partition(List<T> list, Comparator<? super T> cmp, int p, int start, int end) {
|
||||
|
||||
int l = start;
|
||||
int h = end - 2;
|
||||
T piv = list.get(p);
|
||||
swap(list, p, end - 1);
|
||||
|
||||
while (l < h) {
|
||||
if (cmp.compare(list.get(l), piv) <= 0) {
|
||||
l++;
|
||||
} else if (cmp.compare(piv, list.get(h)) <= 0) {
|
||||
h--;
|
||||
} else {
|
||||
swap(list, l, h);
|
||||
}
|
||||
}
|
||||
int idx = h;
|
||||
if (cmp.compare(list.get(h), piv) < 0) {
|
||||
idx++;
|
||||
}
|
||||
swap(list, end - 1, idx);
|
||||
return idx;
|
||||
}
|
||||
|
||||
|
||||
private static <T> void swap(List<T> list, int i, int j) {
|
||||
|
||||
T tmp = list.get(i);
|
||||
list.set(i, list.get(j));
|
||||
list.set(j, tmp);
|
||||
}
|
||||
|
||||
}
|
||||
+6
-11
@@ -1,6 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
@@ -12,29 +13,23 @@ public class Utils {
|
||||
|
||||
private final static float EPSILON = 0.1f;
|
||||
|
||||
|
||||
public static boolean feq(double f1, double f2) {
|
||||
|
||||
return (Math.abs(f1 - f2) < EPSILON);
|
||||
}
|
||||
|
||||
|
||||
public static float round(double d, int decimalPlace) {
|
||||
|
||||
BigDecimal bd = BigDecimal.valueOf(d);
|
||||
bd = bd.setScale(decimalPlace, BigDecimal.ROUND_HALF_UP);
|
||||
return bd.floatValue();
|
||||
}
|
||||
|
||||
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
|
||||
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
|
||||
try {
|
||||
QuickSort.sort(list, comparator);
|
||||
} catch (IllegalArgumentException e) {
|
||||
// This should not happen since we use QuickSort from PDFBox
|
||||
Collections.sort(list, comparator);
|
||||
} catch (IllegalArgumentException e){
|
||||
//TODO Figure out why this happens.
|
||||
log.warn(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+528
@@ -0,0 +1,528 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.IdRemoval;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Status;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationHighlightService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction,
|
||||
ManualRedactions manualRedactions) throws IOException {
|
||||
|
||||
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
|
||||
drawSectionFrames(document, classifiedDoc, flatRedaction, pdPage, page);
|
||||
|
||||
if (classifiedDoc.getEntities().get(page) != null) {
|
||||
addAnnotations(pdPage, classifiedDoc, flatRedaction, manualRedactions, page);
|
||||
}
|
||||
|
||||
if (manualRedactionPages.contains(page)) {
|
||||
addManualAnnotations(pdPage, classifiedDoc, manualRedactions, page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
|
||||
|
||||
Set<Integer> manualRedactionPages = new HashSet<>();
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
manualRedactions.getEntriesToAdd().forEach(entry -> {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
manualRedactionPages.add(pos.getPage());
|
||||
});
|
||||
});
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
|
||||
private void addAnnotations(PDPage pdPage, Document classifiedDoc, boolean flatRedaction,
|
||||
ManualRedactions manualRedactions, int page) throws IOException {
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
|
||||
// Duplicates can exist due table extraction colums over multiple rows.
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
|
||||
entityLoop:
|
||||
for (Entity entity : classifiedDoc.getEntities().get(page)) {
|
||||
|
||||
if (flatRedaction && !isRedactionType(entity)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
boolean requestedToRemove = false;
|
||||
List<Comment> comments = null;
|
||||
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity);
|
||||
if (processedIds.contains(entityPositionSequence.getId())) {
|
||||
|
||||
// TODO refactor this outer loop jump as soon as we have the time.
|
||||
continue entityLoop;
|
||||
} else {
|
||||
processedIds.add(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
|
||||
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
|
||||
if (manualRemoval.getId().equals(entityPositionSequence.getId())) {
|
||||
comments = manualRedactions.getComments().get(manualRemoval.getId());
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
|
||||
entity.setRedaction(false);
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
|
||||
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
|
||||
requestedToRemove = true;
|
||||
manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
|
||||
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
|
||||
.stream()
|
||||
.flatMap(seq -> seq.getTextPositions().stream())
|
||||
.collect(Collectors.toList()), page);
|
||||
|
||||
if (manualRedactions != null) {
|
||||
comments = manualRedactions.getComments().get(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
redactionLogEntry.getPositions().addAll(rectanglesPerLine);
|
||||
|
||||
annotations.addAll(createAnnotation(rectanglesPerLine, entityPositionSequence.getId(), createAnnotationContent(entity), getColor(entity, requestedToRemove), comments, !isHint(entity)));
|
||||
}
|
||||
|
||||
redactionLogEntry.setId(entityPositionSequence.getId());
|
||||
|
||||
// FIXME ids should never be null. Figure out why this happens.
|
||||
if (redactionLogEntry.getId() != null) {
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> getRectanglesPerLine(List<TextPosition> textPositions, int page) {
|
||||
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
if (textPositions.size() == 1) {
|
||||
rectangles.add(new TextPositionSequence(textPositions, page).getRectangle());
|
||||
} else {
|
||||
float y = textPositions.get(0).getYDirAdj();
|
||||
int startIndex = 0;
|
||||
for (int i = 1; i < textPositions.size(); i++) {
|
||||
float yDirAdj = textPositions.get(i).getYDirAdj();
|
||||
if (yDirAdj != y) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, i), page).getRectangle());
|
||||
y = yDirAdj;
|
||||
startIndex = i;
|
||||
}
|
||||
}
|
||||
if (startIndex != textPositions.size() - 1) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
|
||||
}
|
||||
}
|
||||
|
||||
return rectangles;
|
||||
}
|
||||
|
||||
|
||||
private void addManualAnnotations(PDPage pdPage, Document classifiedDoc, ManualRedactions manualRedactions,
|
||||
int page) throws IOException {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
|
||||
|
||||
String id = manualRedactionEntry.getId();
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id);
|
||||
|
||||
List<Rectangle> rectanglesOnPage = new ArrayList<>();
|
||||
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
|
||||
if (page == rectangle.getPage()) {
|
||||
rectanglesOnPage.add(rectangle);
|
||||
redactionLogEntry.getPositions().add(rectangle);
|
||||
}
|
||||
}
|
||||
|
||||
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
|
||||
annotations.addAll(createAnnotation(rectanglesOnPage, id, createAnnotationContent(manualRedactionEntry), getColorForManualAdd(manualRedactionEntry
|
||||
.getType(), manualRedactionEntry.getStatus()), manualRedactions.getComments().get(id), true));
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColor(manualRedactionEntry.getType()))
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.sectionNumber(manualRedactionEntry.getSectionNumber())
|
||||
.manual(true)
|
||||
.status(manualRedactionEntry.getStatus())
|
||||
.manualRedactionType(ManualRedactionType.ADD)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(Entity entity) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.color(getColor(entity, false))
|
||||
.reason(entity.getRedactionReason())
|
||||
.legalBasis(entity.getLegalBasis())
|
||||
.value(entity.getWord())
|
||||
.type(entity.getType())
|
||||
.redacted(entity.isRedaction())
|
||||
.isHint(isHint(entity))
|
||||
.isRecommendation(isRecommendation(entity))
|
||||
.section(entity.getHeadline())
|
||||
.sectionNumber(entity.getSectionNumber())
|
||||
.matchedRule(entity.getMatchedRule())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private List<PDAnnotation> createAnnotation(List<Rectangle> rectangles, String id, String content, float[] color,
|
||||
List<Comment> comments, boolean popup) {
|
||||
|
||||
List<PDAnnotation> annotations = new ArrayList<>();
|
||||
|
||||
PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
|
||||
annotation.constructAppearances();
|
||||
PDRectangle pdRectangle = toPDRectangle(rectangles);
|
||||
annotation.setRectangle(pdRectangle);
|
||||
annotation.setQuadPoints(toQuadPoints(rectangles));
|
||||
if (popup) {
|
||||
annotation.setContents(content);
|
||||
}
|
||||
annotation.setTitlePopup(id);
|
||||
annotation.setAnnotationName(id);
|
||||
annotation.setColor(new PDColor(color, PDDeviceRGB.INSTANCE));
|
||||
annotations.add(annotation);
|
||||
|
||||
if (comments != null) {
|
||||
for (Comment comment : comments) {
|
||||
PDAnnotationText txtAnnot = new PDAnnotationText();
|
||||
txtAnnot.setAnnotationName(comment.getId());
|
||||
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
|
||||
txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
|
||||
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));
|
||||
txtAnnot.setTitlePopup(comment.getUser());
|
||||
txtAnnot.setContents(comment.getText());
|
||||
txtAnnot.setRectangle(pdRectangle);
|
||||
annotations.add(txtAnnot);
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
|
||||
private String createAnnotationContent(Entity entity) {
|
||||
|
||||
return "\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\nLegal basis:" + entity
|
||||
.getLegalBasis() + "\n\nIn section: \"" + entity.getHeadline() + "\"";
|
||||
}
|
||||
|
||||
|
||||
private String createAnnotationContent(ManualRedactionEntry entry) {
|
||||
|
||||
return "\nManual Redaction\n\nIn Section : \"" + entry.getSection() + "\"";
|
||||
}
|
||||
|
||||
|
||||
private PDRectangle toPDRectangle(List<Rectangle> rectangles) {
|
||||
|
||||
float lowerLeftX = Float.MAX_VALUE;
|
||||
float upperRightX = 0;
|
||||
float lowerLeftY = 0;
|
||||
float upperRightY = Float.MAX_VALUE;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
if (rectangle.getTopLeft().getX() < lowerLeftX) {
|
||||
lowerLeftX = rectangle.getTopLeft().getX();
|
||||
}
|
||||
if (rectangle.getTopLeft().getX() + rectangle.getWidth() > upperRightX) {
|
||||
upperRightX = rectangle.getTopLeft().getX() + rectangle.getWidth();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() + rectangle.getHeight() > lowerLeftY) {
|
||||
lowerLeftY = rectangle.getTopLeft().getY() + rectangle.getHeight();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() < upperRightY) {
|
||||
upperRightY = rectangle.getTopLeft().getY();
|
||||
}
|
||||
}
|
||||
|
||||
PDRectangle annotationPosition = new PDRectangle();
|
||||
annotationPosition.setLowerLeftX(lowerLeftX);
|
||||
annotationPosition.setLowerLeftY(lowerLeftY);
|
||||
annotationPosition.setUpperRightX(upperRightX);
|
||||
annotationPosition.setUpperRightY(upperRightY);
|
||||
return annotationPosition;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoints(List<Rectangle> rectangles) {
|
||||
|
||||
float[] quadPoints = new float[rectangles.size() * 8];
|
||||
int i = 0;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
float[] quadPoint = toQuadPoint(rectangle);
|
||||
for (int j = 0; j <= 7; j++) {
|
||||
quadPoints[i + j] = quadPoint[j];
|
||||
}
|
||||
i += 8;
|
||||
}
|
||||
return quadPoints;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoint(Rectangle rectangle) {
|
||||
|
||||
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
|
||||
// of the area to be highlighted
|
||||
return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
|
||||
.getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
|
||||
.getY() + rectangle.getHeight(), rectangle.getTopLeft()
|
||||
.getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
|
||||
}
|
||||
|
||||
|
||||
private boolean isRedactionType(Entity entity) {
|
||||
|
||||
if (!entity.isRedaction()) {
|
||||
return false;
|
||||
}
|
||||
return !isHint(entity);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(Entity entity, boolean requestedToRemove) {
|
||||
|
||||
if (requestedToRemove) {
|
||||
return dictionaryService.getRequestRemoveColor();
|
||||
}
|
||||
if (!entity.isRedaction() && !isHint(entity)) {
|
||||
return dictionaryService.getNotRedactedColor();
|
||||
}
|
||||
return dictionaryService.getColor(entity.getType());
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForManualAdd(String type, Status status) {
|
||||
|
||||
if (status.equals(Status.REQUESTED)) {
|
||||
return dictionaryService.getRequestAddColor();
|
||||
} else if (status.equals(Status.DECLINED)) {
|
||||
return dictionaryService.getNotRedactedColor();
|
||||
}
|
||||
return getColor(type);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type) {
|
||||
return dictionaryService.getColor(type);
|
||||
}
|
||||
|
||||
|
||||
private boolean isHint(Entity entity) {
|
||||
return dictionaryService.isHint(entity.getType());
|
||||
}
|
||||
|
||||
private boolean isRecommendation(Entity entity) {
|
||||
return dictionaryService.isRecommendation(entity.getType());
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionFrames(PDDocument document, Document classifiedDoc, boolean flatRedaction, PDPage pdPage,
|
||||
int page) throws IOException {
|
||||
|
||||
if (flatRedaction) {
|
||||
return;
|
||||
}
|
||||
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
|
||||
AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
|
||||
|
||||
if (textBlock.getPage() != page) {
|
||||
continue;
|
||||
}
|
||||
if (textBlock instanceof TextBlock) {
|
||||
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream);
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size()));
|
||||
|
||||
} else if (textBlock instanceof Table) {
|
||||
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
List<CellRectangle> cellRectangles = visualizeTable((Table) textBlock, contentStream);
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size(), cellRectangles));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
contentStream.close();
|
||||
}
|
||||
|
||||
|
||||
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.LIGHT_GRAY);
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
|
||||
contentStream.stroke();
|
||||
|
||||
if (textBlock.getClassification() != null) {
|
||||
contentStream.beginText();
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
|
||||
contentStream.showText(textBlock.getClassification());
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<CellRectangle> visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
|
||||
|
||||
List<CellRectangle> cellRectangles = new ArrayList<>();
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
if (cell != null) {
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.setStrokingColor(Color.CYAN);
|
||||
contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
|
||||
.getHeight());
|
||||
contentStream.stroke();
|
||||
cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (table.getClassification() != null) {
|
||||
contentStream.beginText();
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
contentStream.newLineAtOffset(table.getMinX(), table.getMinY());
|
||||
contentStream.showText(table.getClassification());
|
||||
contentStream.endText();
|
||||
}
|
||||
|
||||
return cellRectangles;
|
||||
}
|
||||
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PdfFlattenService {
|
||||
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
public PDDocument flattenPDF(PDDocument sourceDoc) throws IOException {
|
||||
|
||||
PDDocument destDoc = new PDDocument();
|
||||
|
||||
PDFRenderer pdfRenderer = new PDFRenderer(sourceDoc);
|
||||
|
||||
final int pageCount = sourceDoc.getDocumentCatalog().getPages().getCount();
|
||||
|
||||
log.info(pageCount + " page" + (pageCount == 1 ? "" : "s") + " to flatten.");
|
||||
|
||||
for (int i = 0; i < pageCount; i += 1) {
|
||||
|
||||
log.info("Flattening page " + (i + 1) + " of " + pageCount + "...");
|
||||
|
||||
BufferedImage img = pdfRenderer.renderImageWithDPI(i, settings.getFlattenImageDpi(), ImageType.RGB);
|
||||
|
||||
log.info("Image rendered in memory (" + img.getWidth() + "x" + img.getHeight() + " " + settings.getFlattenImageDpi() + "DPI). Adding to PDF...");
|
||||
|
||||
PDPage imagePage = new PDPage(new PDRectangle(img.getWidth(), img.getHeight()));
|
||||
destDoc.addPage(imagePage);
|
||||
|
||||
PDImageXObject imgObj = LosslessFactory.createFromImage(destDoc, img);
|
||||
|
||||
PDPageContentStream imagePageContentStream = new PDPageContentStream(destDoc, imagePage);
|
||||
imagePageContentStream.drawImage(imgObj, 0, 0);
|
||||
|
||||
log.info("Image added successfully.");
|
||||
|
||||
imagePageContentStream.close();
|
||||
|
||||
img.flush();
|
||||
}
|
||||
|
||||
log.info("New flattened PDF created in memory.");
|
||||
|
||||
sourceDoc.close();
|
||||
|
||||
return destDoc;
|
||||
}
|
||||
|
||||
}
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
server:
|
||||
port: 8083
|
||||
|
||||
configuration-service.url: "http://localhost:8081"
|
||||
@@ -1,8 +1,6 @@
|
||||
info:
|
||||
description: Redaction Service Server V1
|
||||
|
||||
configuration-service.url: "http://configuration-service-v1:8080"
|
||||
|
||||
server:
|
||||
port: 8080
|
||||
|
||||
|
||||
+51
-132
@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
|
||||
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -45,10 +45,6 @@ import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.IdRemoval;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
@@ -66,7 +62,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(webEnvironment = RANDOM_PORT)
|
||||
@SpringBootTest(webEnvironment = DEFINED_PORT)
|
||||
public class RedactionIntegrationTest {
|
||||
|
||||
private static final String RULES = loadFromClassPath("drools/rules.drl");
|
||||
@@ -80,8 +76,6 @@ public class RedactionIntegrationTest {
|
||||
private static final String MUST_REDACT = "must_redact";
|
||||
private static final String PUBLISHED_INFORMATION = "published_information";
|
||||
private static final String TEST_METHOD = "test_method";
|
||||
private static final String PURITY = "purity";
|
||||
private static final String IMAGE = "image";
|
||||
|
||||
private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
|
||||
private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
|
||||
@@ -104,11 +98,8 @@ public class RedactionIntegrationTest {
|
||||
private final Map<String, Boolean> hintTypeMap = new HashMap<>();
|
||||
private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
|
||||
private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
|
||||
private final Map<String, Integer> rankTypeMap = new HashMap<>();
|
||||
private final Colors colors = new Colors();
|
||||
|
||||
private final static String TEST_RULESET_ID = "123";
|
||||
|
||||
@TestConfiguration
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@@ -134,32 +125,28 @@ public class RedactionIntegrationTest {
|
||||
@Before
|
||||
public void stubClients() {
|
||||
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(RULES));
|
||||
when(rulesClient.getVersion()).thenReturn(0L);
|
||||
when(rulesClient.getRules()).thenReturn(new RulesResponse(RULES));
|
||||
|
||||
loadDictionaryForTest();
|
||||
loadTypeForTest();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(TypeResponse.builder()
|
||||
.types(getTypeResponse())
|
||||
.build());
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SPONSOR));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(HINT_ONLY));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(MUST_REDACT));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(TEST_METHOD));
|
||||
when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PII));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
|
||||
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
|
||||
when(dictionaryClient.getVersion()).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS)).thenReturn(getDictionaryResponse(ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR)).thenReturn(getDictionaryResponse(AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD)).thenReturn(getDictionaryResponse(TEST_METHOD));
|
||||
when(dictionaryClient.getDictionaryForType(PII)).thenReturn(getDictionaryResponse(PII));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
|
||||
when(dictionaryClient.getColors()).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
@@ -235,16 +222,6 @@ public class RedactionIntegrationTest {
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/image.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
|
||||
@@ -270,8 +247,6 @@ public class RedactionIntegrationTest {
|
||||
typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
|
||||
typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
|
||||
typeColorMap.put(FALSE_POSITIVE, "#ffffff");
|
||||
typeColorMap.put(PURITY, "#ffe187");
|
||||
typeColorMap.put(IMAGE, "#fcc5fb");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
@@ -287,8 +262,6 @@ public class RedactionIntegrationTest {
|
||||
hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
hintTypeMap.put(FALSE_POSITIVE, true);
|
||||
hintTypeMap.put(PURITY, false);
|
||||
hintTypeMap.put(IMAGE, true);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
@@ -304,8 +277,6 @@ public class RedactionIntegrationTest {
|
||||
caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
caseInSensitiveMap.put(FALSE_POSITIVE, false);
|
||||
caseInSensitiveMap.put(PURITY, false);
|
||||
caseInSensitiveMap.put(IMAGE, true);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
@@ -321,27 +292,6 @@ public class RedactionIntegrationTest {
|
||||
recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
|
||||
recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
|
||||
recommendationTypeMap.put(FALSE_POSITIVE, false);
|
||||
recommendationTypeMap.put(PURITY, false);
|
||||
recommendationTypeMap.put(IMAGE, false);
|
||||
|
||||
|
||||
rankTypeMap.put(FALSE_POSITIVE, 160);
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
rankTypeMap.put(PII, 150);
|
||||
rankTypeMap.put(ADDRESS, 140);
|
||||
rankTypeMap.put(AUTHOR, 130);
|
||||
rankTypeMap.put(SPONSOR, 120);
|
||||
rankTypeMap.put(VERTEBRATE, 110);
|
||||
rankTypeMap.put(MUST_REDACT, 100);
|
||||
rankTypeMap.put(REDACTION_INDICATOR, 90);
|
||||
rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
|
||||
rankTypeMap.put(PUBLISHED_INFORMATION, 70);
|
||||
rankTypeMap.put(TEST_METHOD, 60);
|
||||
rankTypeMap.put(HINT_ONLY, 50);
|
||||
rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
|
||||
rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
|
||||
rankTypeMap.put(IMAGE, 30);
|
||||
|
||||
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
@@ -356,12 +306,10 @@ public class RedactionIntegrationTest {
|
||||
.stream()
|
||||
.map(typeColor -> TypeResult.builder()
|
||||
.type(typeColor.getKey())
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
.build())
|
||||
|
||||
.collect(Collectors.toList());
|
||||
@@ -376,7 +324,6 @@ public class RedactionIntegrationTest {
|
||||
.isHint(hintTypeMap.get(type))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(type))
|
||||
.isRecommendation(recommendationTypeMap.get(type))
|
||||
.rank(rankTypeMap.get(type))
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -384,7 +331,6 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
System.out.println("noExceptionShouldBeThrownForAnyFiles");
|
||||
ClassLoader loader = getClass().getClassLoader();
|
||||
URL url = loader.getResource("files");
|
||||
@@ -394,12 +340,11 @@ public class RedactionIntegrationTest {
|
||||
input.addAll(getPathsRecursively(file));
|
||||
}
|
||||
for (File path : input) {
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(new FileInputStream(path)))
|
||||
.build();
|
||||
System.out.println("Redacting file : " + path.getName());
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
|
||||
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
|
||||
@@ -411,14 +356,9 @@ public class RedactionIntegrationTest {
|
||||
});
|
||||
}
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
private List<File> getPathsRecursively(File path) {
|
||||
|
||||
List<File> result = new ArrayList<>();
|
||||
@@ -440,32 +380,26 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
// 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
// 182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
// 38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf
|
||||
// 91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
|
||||
// 95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
|
||||
|
||||
|
||||
System.out.println("redactionTest");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
request.setFlatRedaction(false);
|
||||
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
|
||||
if(entry.isRecommendation()){
|
||||
System.out.println(entry.getValue());
|
||||
}
|
||||
});
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
@@ -480,19 +414,14 @@ public class RedactionIntegrationTest {
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
@@ -536,20 +465,15 @@ public class RedactionIntegrationTest {
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.manualRedactions(manualRedactions)
|
||||
.build();
|
||||
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
@@ -598,7 +522,7 @@ public class RedactionIntegrationTest {
|
||||
public void htmlTablesTest() throws IOException {
|
||||
|
||||
System.out.println("htmlTablesTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
|
||||
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
@@ -635,13 +559,12 @@ public class RedactionIntegrationTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Phantom Cells.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
request.setFlatRedaction(false);
|
||||
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
|
||||
if (!entry.isHint()) {
|
||||
@@ -657,19 +580,15 @@ public class RedactionIntegrationTest {
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
request.setFlatRedaction(false);
|
||||
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
|
||||
+123
-103
@@ -1,36 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
@@ -49,8 +20,36 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
|
||||
@SpringBootTest
|
||||
@RunWith(SpringRunner.class)
|
||||
@@ -79,8 +78,6 @@ public class EntityRedactionServiceTest {
|
||||
@Autowired
|
||||
private DroolsExecutionService droolsExecutionService;
|
||||
|
||||
private final static String TEST_RULESET_ID = "123";
|
||||
|
||||
@TestConfiguration
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@@ -107,11 +104,11 @@ public class EntityRedactionServiceTest {
|
||||
public void testNestedEntitiesRemoval() {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
|
||||
entities.add(nested);
|
||||
entities.add(nesting);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
entityRedactionService.removeEntitiesContainedInLarger(entities);
|
||||
|
||||
assertThat(entities.size()).isEqualTo(1);
|
||||
assertThat(entities).contains(nesting);
|
||||
@@ -131,21 +128,21 @@ public class EntityRedactionServiceTest {
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
|
||||
}
|
||||
@@ -164,20 +161,20 @@ public class EntityRedactionServiceTest {
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
|
||||
}
|
||||
@@ -189,22 +186,22 @@ public class EntityRedactionServiceTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
|
||||
" Supplement - Identity of the active substance - Reference list.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()
|
||||
.entrySet()
|
||||
.stream()
|
||||
@@ -214,7 +211,7 @@ public class EntityRedactionServiceTest {
|
||||
"the plant protection product.pdf");
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()
|
||||
.entrySet()
|
||||
.stream()
|
||||
@@ -226,22 +223,22 @@ public class EntityRedactionServiceTest {
|
||||
public void testFalsePositiveInWrongCell() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 9)
|
||||
@@ -290,31 +287,31 @@ public class EntityRedactionServiceTest {
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules(TEST_RULESET_ID);
|
||||
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules();
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 6)
|
||||
.count()).isEqualTo(13);
|
||||
.count()).isEqualTo(18);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -331,27 +328,27 @@ public class EntityRedactionServiceTest {
|
||||
"Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" +
|
||||
".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " +
|
||||
"sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules(TEST_RULESET_ID);
|
||||
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules();
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse authorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(authorResponse);
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(dictionaryResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 11)
|
||||
@@ -370,20 +367,20 @@ public class EntityRedactionServiceTest {
|
||||
.entries(Arrays.asList("Bissig R.", "Thanei P."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
|
||||
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y
|
||||
@@ -395,25 +392,24 @@ public class EntityRedactionServiceTest {
|
||||
.entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(8);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void testNGuideline() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Empty Tabular Data.pdf");
|
||||
@@ -422,20 +418,20 @@ public class EntityRedactionServiceTest {
|
||||
.entries(Collections.singletonList("Aldershof S."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
entityRedactionService.processDocument(classifiedDoc, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6);
|
||||
}
|
||||
@@ -467,27 +463,27 @@ public class EntityRedactionServiceTest {
|
||||
" 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
|
||||
" end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
when(rulesClient.getVersion()).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
|
||||
TypeResponse typeResponse = TypeResponse.builder()
|
||||
.types(Arrays.asList(
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(),
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(),
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build()))
|
||||
TypeResult.builder().type(AUTHOR_CODE).hexColor("#ffff00").build(),
|
||||
TypeResult.builder().type(ADDRESS_CODE).hexColor("#ff00ff").build(),
|
||||
TypeResult.builder().type(SPONSOR_CODE).hexColor("#00ffff").build()))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(typeResponse);
|
||||
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getAllTypes()).thenReturn(typeResponse);
|
||||
|
||||
// Default empty return to prevent NPEs
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
|
||||
|
||||
Colors colors = new Colors();
|
||||
colors.setDefaultColor("#acfc00");
|
||||
@@ -495,7 +491,7 @@ public class EntityRedactionServiceTest {
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
|
||||
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
|
||||
when(dictionaryClient.getColors()).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
@@ -517,4 +513,28 @@ public class EntityRedactionServiceTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAuthorSplitting(){
|
||||
|
||||
String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO";
|
||||
|
||||
word.replaceAll(",", " ").replaceAll(" ", " ");
|
||||
|
||||
Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
|
||||
List<String> allMatches = new ArrayList<>();
|
||||
while (matcher.find()) {
|
||||
allMatches.add(matcher.group());
|
||||
}
|
||||
|
||||
for(String name: allMatches) {
|
||||
if(name.length() >= 3) {
|
||||
System.out.println(name);
|
||||
// dictionaryService.addToLocalDictionary(type, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
public class RegExPatternTest {
|
||||
|
||||
|
||||
@Test
|
||||
public void testExpand(){
|
||||
String pattern = "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)";
|
||||
String text = ", G.R., Simoneaux,";
|
||||
Pattern compiledPattern = Pattern.compile(pattern, 0);
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(1);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testEmailRegEx(){
|
||||
String text = "Address: Schwarzwaldalle " +
|
||||
"P.O.Box\n" +
|
||||
"CH-4002 Basel\n" +
|
||||
"Switzerland\n" +
|
||||
"Contact: Christian Warmers\n" +
|
||||
"Tel: +41 (61) 323 8044\n" +
|
||||
"christian.warmers@syngenta.com";
|
||||
|
||||
|
||||
Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
Matcher matcher = p.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(0);
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testEtAlRegEx() {
|
||||
String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n" +
|
||||
"ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n" +
|
||||
"studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n" +
|
||||
"2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n" +
|
||||
"with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n" +
|
||||
"9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n" +
|
||||
"identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n" +
|
||||
"activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n" +
|
||||
"demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n" +
|
||||
"After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n" +
|
||||
"statistically significant increase of serum hormone concentration was observed for testosterone (at the\n" +
|
||||
"dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n" +
|
||||
"of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n" +
|
||||
"and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n" +
|
||||
"were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n" +
|
||||
"weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n" +
|
||||
"was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n" +
|
||||
"seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n" +
|
||||
"metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n" +
|
||||
"percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n" +
|
||||
"while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n" +
|
||||
"In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n" +
|
||||
"survival and normal morphological and functional development were not altered. No adverse effects\n" +
|
||||
"on male fertility were seen, however important parameters to assess effects on female fertility like\n" +
|
||||
"cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated.";
|
||||
|
||||
Pattern p = Pattern.compile("([^\\s(]*?( \\w\\.?)?) et al\\.?");
|
||||
|
||||
Matcher matcher = p.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(1);
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAuthorSplitting(){
|
||||
|
||||
String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO";
|
||||
|
||||
word.replaceAll(",", " ").replaceAll(" ", " ");
|
||||
|
||||
Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
|
||||
List<String> allMatches = new ArrayList<>();
|
||||
while (matcher.find()) {
|
||||
allMatches.add(matcher.group());
|
||||
}
|
||||
|
||||
for(String name: allMatches) {
|
||||
if(name.length() >= 3) {
|
||||
System.out.println(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class TextNormalizationUtilities {
|
||||
|
||||
/**
|
||||
* Revert hyphenation due to line breaks.
|
||||
* @param text Text to be processed.
|
||||
* @return Text without line-break hyphenation.
|
||||
*/
|
||||
public static String removeHyphenLineBreaks(String text) {
|
||||
return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -10,11 +10,11 @@ public class TextNormalizationUtilitiesTest {
|
||||
|
||||
String test = "Without these peo-\nple, this conference would not happen";
|
||||
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
|
||||
.contains("people");
|
||||
.contains("\npeople");
|
||||
|
||||
test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
|
||||
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
|
||||
.contains("einzelnen", "Vertrages");
|
||||
.contains("\neinzelnen", "\nVertrages");
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
configuration-service.url: "http://configuration-service-v1:8080"
|
||||
|
||||
ribbon:
|
||||
ConnectTimeout: 600000
|
||||
ReadTimeout: 600000
|
||||
|
||||
|
||||
spring:
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
|
||||
+5
-233
@@ -1,238 +1,10 @@
|
||||
(Parent
|
||||
ADME - Bioanalyses, Vergèze,
|
||||
Aeration: Gentle
|
||||
Andrews mean
|
||||
Andrews pKi
|
||||
BASF study code
|
||||
Billington –
|
||||
Bond elute
|
||||
Brown Liquid
|
||||
Brown Orange
|
||||
Brown Solid
|
||||
Brown disco
|
||||
Brown discoloured
|
||||
Brown fat
|
||||
Long-term
|
||||
Brown liquid
|
||||
Brown solid
|
||||
Buehler (1980),
|
||||
Buehler (1980).
|
||||
Buehler 9
|
||||
Buehler Test.
|
||||
Buehler assay
|
||||
Buehler test
|
||||
Buehler)
|
||||
Buehler-like :
|
||||
Calandra lark
|
||||
Cary, NC,
|
||||
Chambersas for
|
||||
Class 4
|
||||
Class III’
|
||||
Class III’)
|
||||
Class method,
|
||||
Class2 (effectively
|
||||
ClassMethod) Vischim
|
||||
Clay =
|
||||
Cochran-Armitage Test.
|
||||
Cochran-armitage test.
|
||||
Dunnett 2
|
||||
Dunnett's "t''
|
||||
Dunnett's Test
|
||||
Dunnett's multiple
|
||||
Dunnett's t-test
|
||||
Dunnett's test
|
||||
Dunnett) **
|
||||
Dunnett) Dose
|
||||
Dunnett) Food
|
||||
Dunnett) Lower
|
||||
Dunnett, 1964)
|
||||
Dunnett’s Multiple
|
||||
Dunnett’s T
|
||||
Dunnett’s Test)
|
||||
Dunnett’s adjustment
|
||||
Dunnett’s multiple
|
||||
Dunnett’s post
|
||||
Dunnett’s t-test)
|
||||
Fine vacuolation
|
||||
Fine white
|
||||
Finewhite powder
|
||||
Fischer 344
|
||||
Fischer 344)
|
||||
Fischer F344
|
||||
Fischer rat
|
||||
Fischer) Contaminants:
|
||||
Fischer-344 acceptability
|
||||
Fischer-344/
|
||||
Fischer-344/DuCrj acceptability
|
||||
Fischer344 rats.
|
||||
Fischer344) :
|
||||
Fischerrat embryo
|
||||
Fischer’s exact
|
||||
Fisher 344
|
||||
Fisher Exact Test
|
||||
Fisher Exact test
|
||||
Fisher exact test
|
||||
Fisher's Exact
|
||||
Fisher-344 rat
|
||||
Fisher344 (F344/
|
||||
Fisher344 rat
|
||||
Fisher344 rats
|
||||
Fisher344 rats.
|
||||
Fisher344 rats/
|
||||
Fisher344) group
|
||||
Fisher’s exact.
|
||||
Freeman-Tukey transform
|
||||
Gentle aeration
|
||||
Grade 2:
|
||||
Grade 3)
|
||||
Grade 3:
|
||||
Green
|
||||
Green Alga
|
||||
Green as
|
||||
Green-, blue-,
|
||||
GreenAlga, Selenastrum
|
||||
GreenAlga.. Springborn
|
||||
Gross External
|
||||
Gross Necropsy
|
||||
Gross and
|
||||
Gross necropsies
|
||||
Gross necropsy
|
||||
Gross pathology
|
||||
Gross pathology:
|
||||
Hall Farm
|
||||
Hall Farm)
|
||||
HallFarm (396
|
||||
Ham's F12
|
||||
Ham’s F10
|
||||
Hand-held
|
||||
HarlanWinkelmann, Borchem,
|
||||
Hill International,
|
||||
Hill Top
|
||||
Hill criteria
|
||||
Hill, Bracknell,
|
||||
HillTop Chamber,
|
||||
Hillmodel was
|
||||
Hill’s model
|
||||
Japanese White
|
||||
Jerneplaque assay
|
||||
Kent. Blood
|
||||
Klimisch score
|
||||
Litchfield and Wilcoxon
|
||||
Long term
|
||||
Long-Term Toxicity
|
||||
Long-term
|
||||
Longbw/
|
||||
Major Macroscopic findings
|
||||
Major biliary
|
||||
Major defects
|
||||
Major deviations
|
||||
Major fetal
|
||||
Major metabolic
|
||||
Major metabolites
|
||||
Major route
|
||||
Major treatment-related
|
||||
Mallard (Anas
|
||||
Mallard with
|
||||
Manual-Hand held
|
||||
Marsh Frog
|
||||
Marsh harrier
|
||||
Masonjars (500
|
||||
Meier formula)
|
||||
Meier survival
|
||||
Miles anti-Canine
|
||||
Miles anti-dog
|
||||
Netherlands, 2011
|
||||
Netherlands, published:
|
||||
Netherlands. Report
|
||||
Penn State
|
||||
Porewater:
|
||||
Rattray study
|
||||
Read across
|
||||
Reed Bunting
|
||||
Reichert-Jung Cryo
|
||||
RifconGmbH. Oxon
|
||||
Ritz and
|
||||
S.P.-U/
|
||||
San Diego,
|
||||
Shaw's Farm,
|
||||
Sprague
|
||||
Strain JapaneseWhite
|
||||
Strong morphological
|
||||
Weight -
|
||||
Weight 2744
|
||||
Weight Gain
|
||||
Weight On
|
||||
Weight Range
|
||||
Weight and
|
||||
Weight and length
|
||||
Weight at
|
||||
Weight gain
|
||||
Weight loss
|
||||
Weight of
|
||||
Weight only
|
||||
Weight range
|
||||
Weight, age
|
||||
Weight, feed
|
||||
Weight-of-evidence approach
|
||||
Manual-Hand held
|
||||
Weight:
|
||||
Weight; Adult
|
||||
Weight; Whole
|
||||
WeightGain 0-1
|
||||
WeightRange:
|
||||
Weightat dosing:
|
||||
Weightrange 200-250g.
|
||||
White (Tif:
|
||||
White 10
|
||||
White 3
|
||||
White Age
|
||||
White Powder
|
||||
White Strain
|
||||
White blood
|
||||
White cell
|
||||
White crystalline
|
||||
White guideline
|
||||
White guinea-pigs
|
||||
White hygroscopic
|
||||
White powder
|
||||
White solid
|
||||
White stork
|
||||
White swan
|
||||
White to off-white
|
||||
White wagtail
|
||||
White, solid
|
||||
WhitePowder Lot/
|
||||
Whiteguideline :
|
||||
Whitehygroscopic solid
|
||||
Whitepowder Lot/
|
||||
Whitepowder Stability
|
||||
Whitesolid Batch
|
||||
Whiteswan)
|
||||
Whiteto off-white
|
||||
Wildlife International specifications
|
||||
Wildlife Internationalspecifications Water:
|
||||
Williams E
|
||||
Williams medium
|
||||
Williams' medium
|
||||
WilliamsE medium
|
||||
Williams’ test
|
||||
Wilson's method.
|
||||
Wilson. All foetal
|
||||
Wilson.All foetal
|
||||
Wood pigeons
|
||||
York.
|
||||
Zeneca dossier);
|
||||
Zenecadossier); see
|
||||
green algae
|
||||
CTL
|
||||
No details reported
|
||||
Not given
|
||||
Not indicated
|
||||
Not reported
|
||||
Not reported.
|
||||
Not stated
|
||||
Syngenta
|
||||
Syngenta AG
|
||||
N/A
|
||||
No details reported
|
||||
Not available
|
||||
Test facility
|
||||
TBD
|
||||
Sprague
|
||||
Weight and length
|
||||
Aeration: Gentle
|
||||
-324
@@ -1,324 +0,0 @@
|
||||
package drools
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
|
||||
|
||||
global Section section
|
||||
|
||||
|
||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||
|
||||
rule "0: Expand CBI Authors with firstname initials"
|
||||
when
|
||||
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
|
||||
then
|
||||
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
||||
section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
||||
end
|
||||
|
||||
|
||||
rule "1: Redact CBI Authors"
|
||||
when
|
||||
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "2: Redacted because Section contains Vertebrate"
|
||||
when
|
||||
Section(matchesType("vertebrate"))
|
||||
then
|
||||
section.redact("CBI_address", 2, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Not Redacted because Section contains no Vertebrate"
|
||||
when
|
||||
Section(!matchesType("vertebrate"))
|
||||
then
|
||||
section.redactNot("CBI_address", 3, "No Vertebrate found");
|
||||
end
|
||||
|
||||
|
||||
rule "4: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
||||
then
|
||||
section.redactNot("CBI_address", 4, "Vertebrate and No Redaction Indicator found");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
||||
then
|
||||
section.redact("CBI_address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
rule "6: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
when
|
||||
Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "7: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
when
|
||||
Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
|
||||
then
|
||||
section.redactCell("Author", 7, "CBI_author", false, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Not redacted because Vertebrate Study = N"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
|
||||
then
|
||||
section.redactCell("Author(s)", 8, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redactNot("CBI_address", 8, "Not redacted because row is not a vertebrate study");
|
||||
section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
|
||||
end
|
||||
|
||||
|
||||
rule "9: Redact if must redact entry is found"
|
||||
when
|
||||
Section(matchesType("must_redact"))
|
||||
then
|
||||
section.redact("CBI_address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
|
||||
then
|
||||
section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "11: Redact sponsor company"
|
||||
when
|
||||
Section(searchText.toLowerCase().contains("batches produced at"))
|
||||
then
|
||||
section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 11, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("batches produced at", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "12: Redact determination of residues"
|
||||
when
|
||||
Section((
|
||||
searchText.toLowerCase.contains("determination of residues") ||
|
||||
searchText.toLowerCase.contains("determination of total residues")
|
||||
) && (
|
||||
searchText.toLowerCase.contains("livestock") ||
|
||||
searchText.toLowerCase.contains("live stock") ||
|
||||
searchText.toLowerCase.contains("tissue") ||
|
||||
searchText.toLowerCase.contains("tissues") ||
|
||||
searchText.toLowerCase.contains("liver") ||
|
||||
searchText.toLowerCase.contains("muscle") ||
|
||||
searchText.toLowerCase.contains("bovine") ||
|
||||
searchText.toLowerCase.contains("ruminant") ||
|
||||
searchText.toLowerCase.contains("ruminants")
|
||||
))
|
||||
then
|
||||
section.redact("CBI_address", 12, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("determination of residues", "must_redact");
|
||||
section.addHintAnnotation("determination of total residues", "must_redact");
|
||||
section.addHintAnnotation("livestock", "must_redact");
|
||||
section.addHintAnnotation("live stock", "must_redact");
|
||||
section.addHintAnnotation("tissue", "must_redact");
|
||||
section.addHintAnnotation("tissues", "must_redact");
|
||||
section.addHintAnnotation("liver", "must_redact");
|
||||
section.addHintAnnotation("muscle", "must_redact");
|
||||
section.addHintAnnotation("bovine", "must_redact");
|
||||
section.addHintAnnotation("ruminant", "must_redact");
|
||||
section.addHintAnnotation("ruminants", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "13: Redact if CTL/* or BL/* was found"
|
||||
when
|
||||
Section(searchText.contains("CTL/") || searchText.contains("BL/"))
|
||||
then
|
||||
section.redact("CBI_address", 13, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("CTL", "must_redact");
|
||||
section.addHintAnnotation("BL", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "14: Redact and add recommendation for et al. author"
|
||||
when
|
||||
Section(searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "15: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
Section(searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
Section(searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
end
|
||||
|
||||
|
||||
rule "17: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("published_information"))
|
||||
then
|
||||
section.redactNot("CBI_author", 17, "Vertebrate and Published Information found");
|
||||
section.redactNot("CBI_address", 17, "Vertebrate and Published Information found");
|
||||
end
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "18: Redacted PII Personal Identification Information"
|
||||
when
|
||||
Section(matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 18, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "19: Redact Emails by RegEx"
|
||||
when
|
||||
Section(searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 19, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "20: Redact contact information"
|
||||
when
|
||||
Section(text.contains("Contact point:")
|
||||
|| text.contains("Phone:")
|
||||
|| text.contains("Fax:")
|
||||
|| text.contains("Tel.:")
|
||||
|| text.contains("Tel:")
|
||||
|| text.contains("E-mail:")
|
||||
|| text.contains("Email:")
|
||||
|| text.contains("e-mail:")
|
||||
|| text.contains("E-mail address:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| text.contains("Telephone number:")
|
||||
|| text.contains("Telephone No:")
|
||||
|| text.contains("Fax number:")
|
||||
|| text.contains("Telephone:")
|
||||
|| text.contains("European contact:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 20, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact contact information if applicant is found"
|
||||
when
|
||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 21, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "22: Redact contact information if Producer is found"
|
||||
when
|
||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
||||
then
|
||||
section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 22, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "23: Redact AUTHOR(S)"
|
||||
when
|
||||
Section(searchText.contains("AUTHOR(S):"))
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 23, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "24: Redact PERFORMING LABORATORY"
|
||||
when
|
||||
Section(searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 24, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "25: Redact On behalf of Sequani Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 25, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "26: Redact On behalf of Syngenta Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 26, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "27: Redact Purity"
|
||||
when
|
||||
Section(searchText.contains("purity"))
|
||||
then
|
||||
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 27, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
|
||||
end
|
||||
+137
-191
@@ -7,58 +7,49 @@ global Section section
|
||||
|
||||
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
||||
|
||||
rule "0: Expand CBI Authors with firstname initials"
|
||||
when
|
||||
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
|
||||
then
|
||||
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
||||
section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
||||
end
|
||||
|
||||
|
||||
rule "1: Redacted because Section contains Vertebrate"
|
||||
when
|
||||
Section(matchesType("vertebrate"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
when
|
||||
Section(matchesType("vertebrate"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "2: Not Redacted because Section contains no Vertebrate"
|
||||
when
|
||||
Section(!matchesType("vertebrate"))
|
||||
then
|
||||
section.redactNot("CBI_author", 2, "No Vertebrate found");
|
||||
section.redactNot("CBI_address", 2, "No Vertebrate found");
|
||||
end
|
||||
when
|
||||
Section(!matchesType("vertebrate"))
|
||||
then
|
||||
section.redactNot("CBI_author", 2, "No Vertebrate found");
|
||||
section.redactNot("CBI_address", 2, "No Vertebrate found");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
||||
then
|
||||
section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
|
||||
section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
|
||||
end
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
||||
then
|
||||
section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
|
||||
section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
|
||||
end
|
||||
|
||||
|
||||
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
||||
then
|
||||
section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
rule "4: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("published_information"))
|
||||
then
|
||||
section.redactNot("CBI_author", 4, "Vertebrate and Published Information found");
|
||||
section.redactNot("CBI_address", 4, "Vertebrate and Published Information found");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("published_information"))
|
||||
then
|
||||
section.redactNot("CBI_author", 5, "Vertebrate and Published Information found");
|
||||
section.redactNot("CBI_address", 5, "Vertebrate and Published Information found");
|
||||
end
|
||||
rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
||||
then
|
||||
section.redact("CBI_author", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "6: Not redacted because Vertebrate Study = N"
|
||||
@@ -73,12 +64,12 @@ rule "6: Not redacted because Vertebrate Study = N"
|
||||
|
||||
|
||||
rule "7: Redact if must redact entry is found"
|
||||
when
|
||||
Section(matchesType("must_redact"))
|
||||
then
|
||||
section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
when
|
||||
Section(matchesType("must_redact"))
|
||||
then
|
||||
section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
@@ -101,91 +92,55 @@ rule "9: Redact sponsor company"
|
||||
|
||||
|
||||
rule "10: Redact determination of residues"
|
||||
when
|
||||
Section((
|
||||
searchText.toLowerCase.contains("determination of residues") ||
|
||||
searchText.toLowerCase.contains("determination of total residues")
|
||||
) && (
|
||||
searchText.toLowerCase.contains("livestock") ||
|
||||
searchText.toLowerCase.contains("live stock") ||
|
||||
searchText.toLowerCase.contains("tissue") ||
|
||||
searchText.toLowerCase.contains("tissues") ||
|
||||
searchText.toLowerCase.contains("liver") ||
|
||||
searchText.toLowerCase.contains("muscle") ||
|
||||
searchText.toLowerCase.contains("bovine") ||
|
||||
searchText.toLowerCase.contains("ruminant") ||
|
||||
searchText.toLowerCase.contains("ruminants")
|
||||
))
|
||||
then
|
||||
section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("determination of residues", "must_redact");
|
||||
section.addHintAnnotation("livestock", "must_redact");
|
||||
section.addHintAnnotation("live stock", "must_redact");
|
||||
section.addHintAnnotation("tissue", "must_redact");
|
||||
section.addHintAnnotation("tissues", "must_redact");
|
||||
section.addHintAnnotation("liver", "must_redact");
|
||||
section.addHintAnnotation("muscle", "must_redact");
|
||||
section.addHintAnnotation("bovine", "must_redact");
|
||||
section.addHintAnnotation("ruminant", "must_redact");
|
||||
section.addHintAnnotation("ruminants", "must_redact");
|
||||
end
|
||||
when
|
||||
Section(searchText.toLowerCase.contains("determination of residues") && (
|
||||
searchText.toLowerCase.contains("livestock") ||
|
||||
searchText.toLowerCase.contains("live stock") ||
|
||||
searchText.toLowerCase.contains("tissue") ||
|
||||
searchText.toLowerCase.contains("liver") ||
|
||||
searchText.toLowerCase.contains("muscle") ||
|
||||
searchText.toLowerCase.contains("bovine") ||
|
||||
searchText.toLowerCase.contains("ruminant") ||
|
||||
searchText.toLowerCase.contains("ruminants")
|
||||
))
|
||||
then
|
||||
section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("determination of residues", "must_redact");
|
||||
section.addHintAnnotation("livestock", "must_redact");
|
||||
section.addHintAnnotation("live stock", "must_redact");
|
||||
section.addHintAnnotation("tissue", "must_redact");
|
||||
section.addHintAnnotation("liver", "must_redact");
|
||||
section.addHintAnnotation("muscle", "must_redact");
|
||||
section.addHintAnnotation("bovine", "must_redact");
|
||||
section.addHintAnnotation("ruminant", "must_redact");
|
||||
section.addHintAnnotation("ruminants", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "11: Redact if CTL/* or BL/* was found"
|
||||
when
|
||||
Section(searchText.contains("CTL/") || searchText.contains("BL/"))
|
||||
then
|
||||
section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("CTL", "must_redact");
|
||||
section.addHintAnnotation("BL", "must_redact");
|
||||
section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("CTL", "must_redact");
|
||||
section.addHintAnnotation("BL", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "12: Redact and add recommendation for et al. author"
|
||||
when
|
||||
Section(searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "13: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
Section(searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
|
||||
rule "14: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
Section(searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
end
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "14: Redacted PII Personal Identification Information"
|
||||
when
|
||||
Section(matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
rule "12: Redacted PII Personal Identification Information"
|
||||
when
|
||||
Section(matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 12, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "15: Redact Emails by RegEx"
|
||||
when
|
||||
Section(searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Redact contact information"
|
||||
rule "13: Redact contact information"
|
||||
when
|
||||
Section(text.contains("Contact point:")
|
||||
|| text.contains("Phone:")
|
||||
@@ -203,105 +158,96 @@ rule "16: Redact contact information"
|
||||
|| text.contains("Telephone:")
|
||||
|| text.contains("European contact:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact point:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 13, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "17: Redact contact information if applicant is found"
|
||||
when
|
||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
rule "14: Redact contact information if applicant is found"
|
||||
when
|
||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Contact:") || text.contains("Telephone number:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 14, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "18: Redact contact information if Producer is found"
|
||||
when
|
||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
||||
then
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
rule "15: Redact contact information if Producer is found"
|
||||
when
|
||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
||||
then
|
||||
section.redactLineAfter("Contact:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 15, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "19: Redact AUTHOR(S)"
|
||||
rule "16: Redact AUTHOR(S)"
|
||||
when
|
||||
Section(searchText.contains("AUTHOR(S):"))
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 16, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "20: Redact PERFORMING LABORATORY"
|
||||
rule "17: Redact PERFORMING LABORATORY"
|
||||
when
|
||||
Section(searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 17, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact On behalf of Sequani Ltd.:"
|
||||
rule "18: Redact On behalf of Sequani Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 18, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "22: Redact On behalf of Syngenta Ltd.:"
|
||||
rule "19: Redact On behalf of Syngenta Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "25: Redact Purity"
|
||||
when
|
||||
Section(searchText.contains("purity"))
|
||||
then
|
||||
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
|
||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 19, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
Reference in New Issue
Block a user