Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 cheng
|
89f0edd304 |
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-parent</artifactId>
|
||||
<version>7.1.2</version>
|
||||
<version>7.0.4</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
|
||||
@@ -34,4 +34,4 @@
|
||||
|
||||
<!-- run 'mvn test' to perform offline validation of the plan -->
|
||||
<!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
|
||||
</project>
|
||||
</project>
|
||||
@@ -51,7 +51,7 @@ public class PlanSpec {
|
||||
private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
|
||||
Permissions permission = new Permissions()
|
||||
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("red-backend", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("gin4", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.loggedInUserPermissions(PermissionType.VIEW)
|
||||
.anonymousUserPermissionView();
|
||||
return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
|
||||
@@ -134,4 +134,4 @@ public class PlanSpec {
|
||||
.whenInactiveInRepositoryAfterDays(14))
|
||||
.notificationForCommitters());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM red/base-image:1.0.0
|
||||
FROM gin5/platform-base:5.2.0
|
||||
|
||||
ARG PLATFORM_JAR
|
||||
|
||||
@@ -6,13 +6,4 @@ ENV PLATFORM_JAR ${PLATFORM_JAR}
|
||||
|
||||
ENV USES_ELASTICSEARCH false
|
||||
|
||||
COPY ["${PLATFORM_JAR}", "/"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
wget cabextract xfonts-utils fonts-liberation \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
&& dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
&& rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
COPY ["${PLATFORM_JAR}", "/"]
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>platform-dependency</artifactId>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<version>1.0.2</version>
|
||||
<version>1.0.1</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<pdfbox.version>2.0.21</pdfbox.version>
|
||||
<pdfbox.version>2.0.16</pdfbox.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@@ -32,21 +32,21 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.1.7</version>
|
||||
<version>1.0.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeRequest {
|
||||
|
||||
private byte[] document;
|
||||
private String ruleSetId;
|
||||
private ManualRedactions manualRedactions;
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeResult {
|
||||
|
||||
private int numberOfPages;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateRequest {
|
||||
|
||||
private byte[] document;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateResponse {
|
||||
|
||||
private byte[] document;
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class CellRectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Comment {
|
||||
|
||||
private String id;
|
||||
private OffsetDateTime date;
|
||||
private String text;
|
||||
private String user;
|
||||
|
||||
}
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class IdRemoval {
|
||||
|
||||
private String id;
|
||||
private String user;
|
||||
private Status status;
|
||||
private boolean removeFromDictionary;
|
||||
|
||||
}
|
||||
-30
@@ -1,30 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class ManualRedactionEntry {
|
||||
|
||||
private String id;
|
||||
private String user;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private String legalBasis;
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private Status status;
|
||||
private boolean addToDictionary;
|
||||
|
||||
private String section;
|
||||
private int sectionNumber;
|
||||
|
||||
}
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ManualRedactionType {
|
||||
ADD, REMOVE
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class ManualRedactions {
|
||||
|
||||
@Builder.Default
|
||||
private Set<IdRemoval> idsToRemove = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Map<String, List<Comment>> comments = new HashMap<>();
|
||||
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Point {
|
||||
|
||||
private float x;
|
||||
private float y;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Rectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
private int page;
|
||||
}
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLog {
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long rulesVersion = -1;
|
||||
|
||||
private String ruleSetId;
|
||||
private String filename;
|
||||
|
||||
|
||||
public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion, String ruleSetId) {
|
||||
|
||||
this.redactionLogEntry = redactionLogEntry;
|
||||
this.dictionaryVersion = dictionaryVersion;
|
||||
this.rulesVersion = rulesVersion;
|
||||
this.ruleSetId = ruleSetId;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
-43
@@ -1,43 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionLogEntry {
|
||||
|
||||
private String id;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private int matchedRule;
|
||||
private String legalBasis;
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<Comment> comments = new ArrayList<>();
|
||||
|
||||
}
|
||||
+1
-2
@@ -12,6 +12,5 @@ import lombok.NoArgsConstructor;
|
||||
public class RedactionRequest {
|
||||
|
||||
private byte[] document;
|
||||
private String ruleSetId;
|
||||
private ManualRedactions manualRedactions;
|
||||
private boolean flatRedaction;
|
||||
}
|
||||
|
||||
-3
@@ -13,7 +13,4 @@ public class RedactionResult {
|
||||
|
||||
private byte[] document;
|
||||
private int numberOfPages;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
|
||||
}
|
||||
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class SectionGrid {
|
||||
|
||||
private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
|
||||
|
||||
}
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public class SectionRectangle {
|
||||
|
||||
@NonNull
|
||||
private Point topLeft;
|
||||
|
||||
@NonNull
|
||||
private float width;
|
||||
|
||||
@NonNull
|
||||
private float height;
|
||||
|
||||
@NonNull
|
||||
private int part;
|
||||
|
||||
@NonNull
|
||||
private int numberOfParts;
|
||||
|
||||
private List<CellRectangle> tableCells;
|
||||
}
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum Status {
|
||||
REQUESTED, APPROVED, DECLINED
|
||||
}
|
||||
+10
-19
@@ -1,28 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
String SERVICE_NAME = "redaction-service-v1";
|
||||
|
||||
String RULE_SET_PARAMETER_NAME = "ruleSetId";
|
||||
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
|
||||
|
||||
@PostMapping(value = "/analyze", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest);
|
||||
|
||||
@PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
|
||||
@PostMapping(value = "/redact", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult redact(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
|
||||
@@ -33,10 +24,10 @@ public interface RedactionResource {
|
||||
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/rules/update"+RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId);
|
||||
@GetMapping(value = "/rules", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
String getRules();
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void testRules(@RequestBody String rules);
|
||||
@PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@RequestBody String rules);
|
||||
|
||||
}
|
||||
@@ -11,6 +11,25 @@
|
||||
|
||||
<artifactId>redaction-service-server-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<pdfbox.version>2.0.20</pdfbox.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
@@ -20,7 +39,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>0.10.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.drools</groupId>
|
||||
@@ -37,22 +56,17 @@
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>1.16.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>29.0-jre</version>
|
||||
</dependency>
|
||||
<!-- commons -->
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<groupId>com.iqser.gin4.commons</groupId>
|
||||
<artifactId>spring-commons</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<groupId>com.iqser.gin4.commons</groupId>
|
||||
<artifactId>logging-commons</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<groupId>com.iqser.gin4.commons</groupId>
|
||||
<artifactId>metric-commons</artifactId>
|
||||
</dependency>
|
||||
<!-- other external -->
|
||||
@@ -85,7 +99,7 @@
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<groupId>com.iqser.gin4.commons</groupId>
|
||||
<artifactId>test-commons</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
+27
-7
@@ -1,26 +1,46 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.internal.io.ResourceFactory;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
|
||||
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
@Import({DefaultWebMvcConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
|
||||
public class Application {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
private static final String drlFile = "drools/rules.drl";
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
kieFileSystem.write(ResourceFactory.newClassPathResource(drlFile));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+2
-8
@@ -4,9 +4,8 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
import lombok.Data;
|
||||
@@ -18,15 +17,10 @@ public class Document {
|
||||
|
||||
private List<Page> pages = new ArrayList<>();
|
||||
private List<Paragraph> paragraphs = new ArrayList<>();
|
||||
private Map<Integer, List<Entity>> entities = new HashMap<>();
|
||||
private Map<Integer, Set<Entity>> entities = new HashMap<>();
|
||||
private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
|
||||
private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
|
||||
private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
|
||||
private boolean headlines;
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
private SectionGrid sectionGrid = new SectionGrid();
|
||||
private long dictionaryVersion;
|
||||
private long rulesVersion;
|
||||
}
|
||||
|
||||
-3
@@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
@@ -17,8 +16,6 @@ public class Page {
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
|
||||
private List<Rectangle2D> imageBounds;
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
private boolean landscape;
|
||||
|
||||
+6
-21
@@ -10,19 +10,17 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class Paragraph {
|
||||
|
||||
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
|
||||
private String headline;
|
||||
|
||||
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
public SearchableText getSearchableText(){
|
||||
SearchableText searchableText = new SearchableText();
|
||||
pageBlocks.forEach(block -> {
|
||||
if (block instanceof TextBlock) {
|
||||
if(block instanceof TextBlock){
|
||||
searchableText.addAll(((TextBlock) block).getSequences());
|
||||
}
|
||||
});
|
||||
@@ -30,27 +28,14 @@ public class Paragraph {
|
||||
}
|
||||
|
||||
|
||||
public List<Table> getTables() {
|
||||
|
||||
public List<Table> getTables(){
|
||||
List<Table> tables = new ArrayList<>();
|
||||
pageBlocks.forEach(block -> {
|
||||
if (block instanceof Table) {
|
||||
if(block instanceof Table){
|
||||
tables.add((Table) block);
|
||||
}
|
||||
});
|
||||
return tables;
|
||||
}
|
||||
|
||||
|
||||
public List<TextBlock> getTextBlocks() {
|
||||
|
||||
List<TextBlock> textBlocks = new ArrayList<>();
|
||||
pageBlocks.forEach(block -> {
|
||||
if (block instanceof TextBlock) {
|
||||
textBlocks.add((TextBlock) block);
|
||||
}
|
||||
});
|
||||
return textBlocks;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+16
-18
@@ -5,45 +5,43 @@ import java.util.Map;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class StringFrequencyCounter {
|
||||
|
||||
@Getter
|
||||
private final Map<String, Integer> countPerValue = new HashMap<>();
|
||||
Map<String, Integer> countPerValue = new HashMap<>();
|
||||
|
||||
|
||||
public void add(String value) {
|
||||
|
||||
if (!countPerValue.containsKey(value)) {
|
||||
public void add(String value){
|
||||
if(!countPerValue.containsKey(value)){
|
||||
countPerValue.put(value, 1);
|
||||
} else {
|
||||
countPerValue.put(value, countPerValue.get(value) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addAll(Map<String, Integer> otherCounter) {
|
||||
|
||||
for (Map.Entry<String, Integer> entry : otherCounter.entrySet()) {
|
||||
if (countPerValue.containsKey(entry.getKey())) {
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
|
||||
public void addAll(Map<String, Integer> otherCounter){
|
||||
for(Map.Entry<String, Integer> entry: otherCounter.entrySet()){
|
||||
if(countPerValue.containsKey(entry.getKey())){
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey())+ entry.getValue());
|
||||
} else {
|
||||
countPerValue.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public String getMostPopular() {
|
||||
|
||||
public String getMostPopular(){
|
||||
Map.Entry<String, Integer> mostPopular = null;
|
||||
for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
|
||||
if (mostPopular == null) {
|
||||
for(Map.Entry<String, Integer> entry: countPerValue.entrySet()){
|
||||
if(mostPopular == null){
|
||||
mostPopular = entry;
|
||||
} else if (entry.getValue() > mostPopular.getValue()) {
|
||||
} else if(entry.getValue() > mostPopular.getValue()){
|
||||
mostPopular = entry;
|
||||
}
|
||||
}
|
||||
return mostPopular != null ? mostPopular.getKey() : null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+7
-1
@@ -97,7 +97,13 @@ public class TextBlock extends AbstractTextContainer {
|
||||
this.maxY = Math.max(y1, y2);
|
||||
}
|
||||
|
||||
public float getHeight() {
|
||||
return maxY - minY;
|
||||
}
|
||||
|
||||
public float getWidth() {
|
||||
return maxX - minX;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
@@ -125,7 +131,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
if (Math.abs(previous.getY1() - word.getY1()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
|
||||
+19
-13
@@ -29,16 +29,20 @@ public class BlockificationService {
|
||||
float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
|
||||
TextPositionSequence prev = null;
|
||||
|
||||
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
|
||||
boolean startFromTop = word.getY1() > maxY + word.getHeight();
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), verticalRulingLines))) {
|
||||
if (prev != null &&
|
||||
(lineSeparation
|
||||
|| startFromTop
|
||||
|| word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines)
|
||||
|| word.getRotation() == 0 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines)
|
||||
|| word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines)
|
||||
|| word.getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines)
|
||||
)) {
|
||||
|
||||
TextBlock cb1 = buildTextBlock(chunkWords);
|
||||
chunkBlockList1.add(cb1);
|
||||
@@ -96,12 +100,11 @@ public class BlockificationService {
|
||||
styleFrequencyCounter.add(wordBlock.getFontStyle());
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
|
||||
.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
|
||||
} else {
|
||||
TextBlock spatialEntity = textBlock.union(wordBlock);
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
|
||||
.getHeight());
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
|
||||
spatialEntity.getWidth(), spatialEntity.getHeight());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +122,6 @@ public class BlockificationService {
|
||||
|
||||
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
|
||||
|
||||
for (Ruling ruling : rulingLines) {
|
||||
if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
|
||||
return true;
|
||||
@@ -131,6 +133,7 @@ public class BlockificationService {
|
||||
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
float minY = 10000;
|
||||
@@ -144,6 +147,7 @@ public class BlockificationService {
|
||||
|
||||
for (AbstractTextContainer container : page.getTextBlocks()) {
|
||||
|
||||
|
||||
if (container instanceof TextBlock) {
|
||||
TextBlock textBlock = (TextBlock) container;
|
||||
if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
|
||||
@@ -175,15 +179,16 @@ public class BlockificationService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (container instanceof Table) {
|
||||
Table table = (Table) container;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
for (Cell column : row) {
|
||||
|
||||
if (cell == null || cell.getTextBlocks() == null) {
|
||||
if (column == null || column.getTextBlocks() == null) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
for (TextBlock textBlock : column.getTextBlocks()) {
|
||||
if (textBlock.getMinX() < minX) {
|
||||
minX = textBlock.getMinX();
|
||||
}
|
||||
@@ -206,4 +211,5 @@ public class BlockificationService {
|
||||
return new Rectangle(minY, minX, maxX - minX, maxY - minY);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+18
-44
@@ -5,17 +5,15 @@ import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ClassificationService {
|
||||
@@ -30,7 +28,7 @@ public class ClassificationService {
|
||||
|
||||
List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
|
||||
|
||||
log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
|
||||
System.out.println(document.getFontSizeCounter().getCountPerValue());
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
@@ -41,7 +39,6 @@ public class ClassificationService {
|
||||
|
||||
|
||||
public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
|
||||
@@ -50,33 +47,24 @@ public class ClassificationService {
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// TODO Figure out why this happens.
|
||||
return;
|
||||
}
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Header");
|
||||
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1
|
||||
&& (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
|
||||
&& PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@@ -84,34 +72,20 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getSequences().size() > 3){
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-2
@@ -1,8 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface DictionaryClient extends DictionaryResource {
|
||||
}
|
||||
+3
-2
@@ -1,8 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface RulesClient extends RulesResource {
|
||||
}
|
||||
+1
-1
@@ -2,13 +2,13 @@ package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import com.iqser.red.commons.spring.ErrorMessage;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
import com.iqser.gin4.commons.api.errorhandling.ErrorMessage;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
+61
-98
@@ -1,112 +1,80 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final PdfVisualisationService pdfVisualisationService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final RedactionLogCreatorService redactionLogCreatorService;
|
||||
private final AnnotationHighlightService annotationHighlightService;
|
||||
private final EntityRedactionService entityRedactionService;
|
||||
private final PdfFlattenService pdfFlattenService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final DictionaryService dictionaryService;
|
||||
private final AnnotationService annotationService;
|
||||
private final RulesClient rulesClient;
|
||||
|
||||
private long rulesVersion = -1;
|
||||
|
||||
@Override
|
||||
public AnalyzeResult analyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(analyzeRequest.getDocument()))) {
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
updateRules();
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc);
|
||||
annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction());
|
||||
|
||||
log.info("Document structure analysis successful, starting redaction analysis...");
|
||||
|
||||
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getRuleSetId(), analyzeRequest.getManualRedactions());
|
||||
redactionLogCreatorService.createRedactionLog(classifiedDoc, pdDocument.getNumberOfPages(), analyzeRequest.getManualRedactions(), analyzeRequest
|
||||
.getRuleSetId());
|
||||
|
||||
log.info("Redaction analysis successful...");
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.sectionGrid(classifiedDoc.getSectionGrid())
|
||||
.redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc
|
||||
.getRulesVersion(), analyzeRequest.getRuleSetId()))
|
||||
.numberOfPages(classifiedDoc.getPages().size())
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(annotateRequest.getDocument()))) {
|
||||
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
dictionaryService.updateDictionary(annotateRequest.getRedactionLog().getRuleSetId());
|
||||
annotationService.annotate(pdDocument, annotateRequest.getRedactionLog(), annotateRequest.getSectionGrid());
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
pdDocument.save(byteArrayOutputStream);
|
||||
return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
|
||||
if (redactionRequest.isFlatRedaction()) {
|
||||
PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
|
||||
return convert(flatDocument, classifiedDoc.getPages().size());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
|
||||
updateRules();
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size(), pdfSegmentationRequest.getRuleSetId());
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -114,17 +82,17 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
updateRules();
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size(), redactionRequest.getRuleSetId());
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -132,11 +100,11 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
updateRules();
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
@@ -159,41 +127,36 @@ public class RedactionController implements RedactionResource {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRules() {
|
||||
return droolsExecutionService.getRules();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String ruleSetId) {
|
||||
|
||||
droolsExecutionService.updateRules(ruleSetId);
|
||||
public void updateRules(@RequestBody String rules) {
|
||||
droolsExecutionService.updateRules(rules);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void testRules(@RequestBody String rules) {
|
||||
|
||||
droolsExecutionService.testRules(rules);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages, String ruleSetId) throws IOException {
|
||||
|
||||
return convert(document, numberOfPages, null, null, 0, 0, ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages,
|
||||
List<RedactionLogEntry> redactionLogEntities, SectionGrid sectionGrid,
|
||||
long dictionaryVersion, long rulesVersion, String ruleSetId) throws IOException {
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
document.save(byteArrayOutputStream);
|
||||
return RedactionResult.builder()
|
||||
.document(byteArrayOutputStream.toByteArray())
|
||||
.numberOfPages(numberOfPages)
|
||||
.redactionLog(new RedactionLog(redactionLogEntities, dictionaryVersion, rulesVersion, ruleSetId))
|
||||
.sectionGrid(sectionGrid)
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
private void updateRules() {
|
||||
|
||||
long version = rulesClient.getVersion();
|
||||
if (version > rulesVersion) {
|
||||
rulesVersion = version;
|
||||
updateRules(rulesClient.getRules().getRules());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+26
-118
@@ -1,8 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -30,15 +28,10 @@ import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
@@ -50,11 +43,11 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
@Setter
|
||||
protected PDPage pdpage;
|
||||
@Getter
|
||||
private float minCharWidth = Float.MAX_VALUE;
|
||||
|
||||
@Getter
|
||||
private int minCharWidths;
|
||||
private float minCharHeight = Float.MAX_VALUE;
|
||||
|
||||
@Getter
|
||||
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
@@ -64,18 +57,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private List<Rectangle2D> imageBounds = new ArrayList<>();
|
||||
|
||||
private float path_x;
|
||||
private float path_y;
|
||||
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
|
||||
public PDFLinesTextStripper() throws IOException {
|
||||
|
||||
super();
|
||||
this.addOperator(new SetStrokingColorSpace());
|
||||
this.addOperator(new SetNonStrokingColorSpace());
|
||||
@@ -99,9 +87,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
this.addOperator(new SetLineWidth());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments)
|
||||
throws IOException {
|
||||
|
||||
String operation = operator.getName();
|
||||
|
||||
@@ -122,11 +110,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
// The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
|
||||
if (pos.getY() > path_y) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
|
||||
.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
|
||||
.getX(), path_y)));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
|
||||
}
|
||||
|
||||
path_x = (float) pos.getX();
|
||||
@@ -147,25 +133,19 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
Point2D p2 = transformPosition(x + width, y + height);
|
||||
|
||||
// Horizontal lines
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
|
||||
// Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
}
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -188,59 +168,12 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
case OperatorName.ENDPATH:
|
||||
graphicsPath.clear();
|
||||
break;
|
||||
|
||||
case OperatorName.DRAW_OBJECT:
|
||||
processImageOperation(arguments);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
super.processOperator(operator, arguments);
|
||||
}
|
||||
|
||||
|
||||
protected void processImageOperation(List<COSBase> arguments) {
|
||||
|
||||
try {
|
||||
COSName objectName = (COSName) arguments.get(0);
|
||||
PDXObject xobject = getResources().getXObject(objectName);
|
||||
if (xobject instanceof PDImageXObject) {
|
||||
PDImageXObject pdfImage = (PDImageXObject) xobject;
|
||||
|
||||
Rectangle2D imageBounds = calculateImagePosition(pdfImage);
|
||||
|
||||
Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
|
||||
.getWidth(), (float) imageBounds.getHeight());
|
||||
|
||||
if (rect.getHeight() > 2 && rect.getWidth() > 2) {
|
||||
this.imageBounds.add(rect);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Problem during image extraction: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
|
||||
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
|
||||
Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
|
||||
|
||||
AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
|
||||
imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
|
||||
imageTransform.translate(0, -pdfImage.getHeight());
|
||||
|
||||
AffineTransform pageTransform = new AffineTransform();
|
||||
pageTransform.concatenate(imageTransform);
|
||||
|
||||
return pageTransform.createTransformedShape(imageBounds).getBounds2D();
|
||||
}
|
||||
|
||||
|
||||
private float floatValue(COSBase value) {
|
||||
|
||||
if (value instanceof COSNumber) {
|
||||
return ((COSNumber) value).floatValue();
|
||||
} else {
|
||||
@@ -248,45 +181,30 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Point2D.Float transformPosition(float x, float y) {
|
||||
|
||||
return super.transformedPoint(x, y);
|
||||
}
|
||||
|
||||
|
||||
private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {
|
||||
|
||||
try {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
|
||||
.toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor()
|
||||
.isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
rulings.addAll(path);
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " or " + getGraphicsState().getNonStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " does not support toRGB");
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
minCharWidth = Math.min(minCharWidth, textPositions.get(i).getWidthDirAdj());
|
||||
minCharHeight = Math.min(minCharHeight, textPositions.get(i).getHeightDir());
|
||||
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if (charWidth < minCharWidths) {
|
||||
minCharWidths = charWidth;
|
||||
}
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0"))) {
|
||||
if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
@@ -294,21 +212,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")) && i <= textPositions.size() - 2) {
|
||||
if (i > 0 && textPositions.get(i).getUnicode().equals(" ") && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i + 1;
|
||||
@@ -316,26 +228,22 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
|
||||
if (!sublist.isEmpty() && sublist.get(sublist.size() - 1).getUnicode().equals(" ")) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText(PDDocument doc) throws IOException {
|
||||
|
||||
minCharWidths = Integer.MAX_VALUE;
|
||||
minCharWidth = Float.MAX_VALUE;
|
||||
minCharHeight = Float.MAX_VALUE;
|
||||
textPositionSequences.clear();
|
||||
imageBounds = new ArrayList<>();
|
||||
rulings.clear();
|
||||
graphicsPath.clear();
|
||||
path_x = 0.0f;
|
||||
|
||||
+1
-2
@@ -1,6 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
@@ -14,10 +13,10 @@ public class ParsedElements {
|
||||
|
||||
private List<TextPositionSequence> sequences;
|
||||
private List<Ruling> rulings;
|
||||
private List<Rectangle2D> imageBounds;
|
||||
|
||||
private boolean landscape;
|
||||
private boolean rotated;
|
||||
|
||||
private float minCharWidth;
|
||||
private float minCharHeight;
|
||||
}
|
||||
|
||||
+10
-87
@@ -5,11 +5,10 @@ import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Setter;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
@@ -17,50 +16,36 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
private List<TextPosition> textPositions = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
private float[] annotationColor;
|
||||
|
||||
private final int page;
|
||||
|
||||
|
||||
public TextPositionSequence(List<TextPosition> textPositions, int page) {
|
||||
|
||||
public TextPositionSequence(List<TextPosition> textPositions, int page){
|
||||
this.textPositions = textPositions;
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
|
||||
return textPositions.size();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return text.charAt(0);
|
||||
}
|
||||
|
||||
|
||||
public char charAt(int index, boolean caseInSensitive) {
|
||||
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextPositionSequence subSequence(int start, int end) {
|
||||
|
||||
return new TextPositionSequence(textPositions.subList(start, end), page);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder builder = new StringBuilder(length());
|
||||
for (int i = 0; i < length(); i++) {
|
||||
builder.append(charAt(i));
|
||||
@@ -68,21 +53,15 @@ public class TextPositionSequence implements CharSequence {
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
|
||||
public TextPosition textPositionAt(int index) {
|
||||
|
||||
return textPositions.get(index);
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPosition textPosition) {
|
||||
|
||||
this.textPositions.add(textPosition);
|
||||
}
|
||||
|
||||
|
||||
public float getX1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
} else {
|
||||
@@ -90,23 +69,15 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public float getX2() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
} else {
|
||||
return textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
}
|
||||
}
|
||||
|
||||
public float getRotationAdjustedY() {
|
||||
return textPositions.get(0).getY();
|
||||
}
|
||||
|
||||
public float getY1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
} else {
|
||||
@@ -114,46 +85,30 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public float getY2() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() -2 ;
|
||||
} else {
|
||||
return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public float getTextHeight() {
|
||||
|
||||
return textPositions.get(0).getHeightDir() + 2;
|
||||
}
|
||||
|
||||
|
||||
public float getHeight() {
|
||||
|
||||
return getY2() - getY1();
|
||||
}
|
||||
|
||||
|
||||
public float getWidth() {
|
||||
|
||||
return getX2() - getX1();
|
||||
}
|
||||
|
||||
|
||||
public String getFont() {
|
||||
|
||||
return textPositions.get(0)
|
||||
.getFont()
|
||||
.toString()
|
||||
.toLowerCase()
|
||||
.replaceAll(",bold", "")
|
||||
.replaceAll(",italic", "");
|
||||
return textPositions.get(0).getFont().toString().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
|
||||
}
|
||||
|
||||
|
||||
public String getFontStyle() {
|
||||
|
||||
String lowercaseFontName = textPositions.get(0).getFont().toString().toLowerCase();
|
||||
@@ -170,48 +125,16 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
}
|
||||
|
||||
|
||||
public float getFontSize() {
|
||||
|
||||
return textPositions.get(0).getFontSizeInPt();
|
||||
}
|
||||
|
||||
|
||||
public float getSpaceWidth() {
|
||||
|
||||
return textPositions.get(0).getWidthOfSpace();
|
||||
}
|
||||
|
||||
|
||||
public int getRotation() {
|
||||
|
||||
return textPositions.get(0).getRotation();
|
||||
}
|
||||
|
||||
|
||||
public Rectangle getRectangle() {
|
||||
|
||||
float height = getTextHeight();
|
||||
|
||||
float posXInit = getX1();
|
||||
float posXEnd;
|
||||
float posYInit;
|
||||
float posYEnd;
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
posXEnd = textPositions.get(0).getYDirAdj() + 2;
|
||||
posYInit = getY1();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
|
||||
} else {
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + 2;
|
||||
}
|
||||
|
||||
return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
-50
@@ -1,50 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Value;
|
||||
|
||||
@Value
|
||||
public class CellValue {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
private int rowSpanStart;
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
while (itty.hasNext()) {
|
||||
TextBlock textBlock = itty.next();
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
if (itty.hasNext()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
}
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
@Data
|
||||
public class Dictionary {
|
||||
|
||||
public static final String RECOMMENDATION_PREFIX = "recommendation_";
|
||||
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionaryModels;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
@Getter
|
||||
private long version;
|
||||
|
||||
|
||||
public Dictionary(List<DictionaryModel> dictionaryModels, long dictionaryVersion){
|
||||
this.dictionaryModels = dictionaryModels;
|
||||
this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
this.version = dictionaryVersion;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type){
|
||||
if(!localAccessMap.containsKey(type)){
|
||||
return 0;
|
||||
}
|
||||
return localAccessMap.get(type).getRank();
|
||||
}
|
||||
|
||||
|
||||
public boolean isRecommendation(String type) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isRecommendation();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasLocalEntries() {
|
||||
|
||||
return dictionaryModels.stream().anyMatch(dm -> !dm.getLocalEntries().isEmpty());
|
||||
}
|
||||
|
||||
|
||||
public Set<String> getTypes() {
|
||||
|
||||
return localAccessMap.keySet();
|
||||
}
|
||||
|
||||
|
||||
public boolean containsValue(String type, String value) {
|
||||
|
||||
if (localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getLocalEntries()
|
||||
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
|
||||
.getLocalEntries()
|
||||
.contains(value)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public boolean isHint(String type) {
|
||||
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isHint();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isCaseInsensitiveDictionary(String type) {
|
||||
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type);
|
||||
if (dictionaryModel != null) {
|
||||
return dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryModel implements Serializable {
|
||||
|
||||
private String type;
|
||||
private int rank;
|
||||
private float[] color;
|
||||
private boolean caseInsensitive;
|
||||
private boolean hint;
|
||||
private boolean recommendation;
|
||||
private Set<String> entries;
|
||||
private Set<String> localEntries;
|
||||
|
||||
public Set<String> getValues(boolean local){
|
||||
return local ? localEntries : entries;
|
||||
}
|
||||
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
public class DictionaryRepresentation {
|
||||
|
||||
private String ruleSetId;
|
||||
private long dictionaryVersion = -1;
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
private float[] defaultColor;
|
||||
private float[] requestAddColor;
|
||||
private float[] requestRemoveColor;
|
||||
private float[] notRedactedColor;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
|
||||
|
||||
}
|
||||
+3
-39
@@ -1,70 +1,34 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity {
|
||||
|
||||
|
||||
private final String word;
|
||||
private final String type;
|
||||
private boolean redaction;
|
||||
private String redactionReason;
|
||||
private String legalBasis;
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer end;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private String headline;
|
||||
private int matchedRule;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) {
|
||||
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences) {
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
this.redaction = redaction;
|
||||
this.redactionReason = redactionReason;
|
||||
this.positionSequences = positionSequences;
|
||||
this.headline = headline;
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
this.textAfter = textAfter;
|
||||
}
|
||||
|
||||
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry) {
|
||||
|
||||
public Entity(String word, String type, Integer start, Integer end) {
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-7
@@ -2,23 +2,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class EntityPositionSequence {
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
private int pageNumber;
|
||||
private final String id;
|
||||
|
||||
private final UUID id;
|
||||
}
|
||||
|
||||
+37
-93
@@ -1,98 +1,57 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
@SuppressWarnings("all")
|
||||
public class SearchableText {
|
||||
|
||||
private final List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
|
||||
public void add(TextPositionSequence textPositionSequence) {
|
||||
|
||||
sequences.add(textPositionSequence);
|
||||
}
|
||||
|
||||
|
||||
public void addAll(List<TextPositionSequence> textPositionSequences) {
|
||||
|
||||
sequences.addAll(textPositionSequences);
|
||||
}
|
||||
|
||||
|
||||
public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {
|
||||
public List<EntityPositionSequence> getSequences(String searchString) {
|
||||
|
||||
return getSequences(searchString, caseInsensitive, null);
|
||||
|
||||
}
|
||||
|
||||
@SuppressWarnings("checkstyle:ModifiedControlVariable")
|
||||
public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
|
||||
List<TextPositionSequence> sequencesSubList) {
|
||||
|
||||
String normalizedSearchString;
|
||||
if (caseInsensitive) {
|
||||
normalizedSearchString = searchString.toLowerCase();
|
||||
} else {
|
||||
normalizedSearchString = searchString;
|
||||
}
|
||||
|
||||
char[] searchChars = normalizedSearchString.replaceAll("\\n", " ").toCharArray();
|
||||
char[] searchChars = searchString.replaceAll("\\n", " ").toCharArray();
|
||||
int counter = 0;
|
||||
|
||||
|
||||
List<TextPositionSequence> crossSequenceParts = new ArrayList<>();
|
||||
List<EntityPositionSequence> finalMatches = new ArrayList<>();
|
||||
for (int i = 0; i < sequences.size(); i++) {
|
||||
TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
|
||||
for (int j = 0; j < sequences.get(i).length(); j++) {
|
||||
|
||||
List<TextPositionSequence> searchSpace;
|
||||
if (sequencesSubList != null) {
|
||||
int subListIndex = Collections.indexOfSubList(sequences, sequencesSubList);
|
||||
if (subListIndex != -1) {
|
||||
searchSpace = sequences.subList(subListIndex, subListIndex + sequencesSubList.size());
|
||||
} else {
|
||||
searchSpace = sequences;
|
||||
}
|
||||
} else {
|
||||
searchSpace = sequences;
|
||||
}
|
||||
|
||||
for (int i = 0; i < searchSpace.size(); i++) {
|
||||
TextPositionSequence partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
|
||||
for (int j = 0; j < searchSpace.get(i).length(); j++) {
|
||||
|
||||
if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
|
||||
if(i > 0 && j == 0 && sequences.get(i).charAt(0) == ' ' && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) == ' '
|
||||
|| j > 0 && sequences.get(i).charAt(j) == ' ' && sequences.get(i).charAt(j - 1) == ' '){
|
||||
if(j == sequences.get(i).length() -1 && counter != 0 && !partMatch.getTextPositions().isEmpty()){
|
||||
crossSequenceParts.add(partMatch);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (j == 0 && searchSpace.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1)
|
||||
.length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
|
||||
if(j == 0 && sequences.get(i).charAt(j) != ' ' && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && searchChars[counter] == ' '){
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == '-') {
|
||||
if (sequences.get(i).charAt(j) == searchChars[counter] || counter != 0 && sequences.get(i).charAt(j) == '-') {
|
||||
|
||||
if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
|
||||
.charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1)
|
||||
.length() - 1, caseInsensitive)) || j == 0 && i != 0 && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) != ' ' && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) != ' ') {
|
||||
partMatch.add(searchSpace.get(i).textPositionAt(j));
|
||||
if (!(j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
|
||||
if(counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i).charAt(j - 1)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1))
|
||||
|| j == 0 && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && sequences.get(i).charAt(j) != ' ') {
|
||||
partMatch.add(sequences.get(i).textPositionAt(j));
|
||||
if (!(j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) == '-' && searchChars[counter] != '-')) {
|
||||
counter++;
|
||||
}
|
||||
}
|
||||
@@ -100,63 +59,53 @@ public class SearchableText {
|
||||
if (counter == searchString.length()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i)
|
||||
.charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive) != ' ') {
|
||||
if(i == sequences.size() - 1 && j == sequences.get(i).length() -1
|
||||
|| j != sequences.get(i).length() -1 && isSeparator(sequences.get(i).charAt(j +1))
|
||||
|| j == sequences.get(i).length() -1 && isSeparator(sequences.get(i + 1).charAt(0))
|
||||
|| j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) != ' ' && sequences.get(i + 1).charAt(0) != ' ') {
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
crossSequenceParts = new ArrayList<>();
|
||||
partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
|
||||
partMatch = new TextPositionSequence(sequences.get(i).getPage());
|
||||
}
|
||||
} else {
|
||||
counter = 0;
|
||||
if (!crossSequenceParts.isEmpty()) {
|
||||
if(!crossSequenceParts.isEmpty()){
|
||||
j--;
|
||||
}
|
||||
crossSequenceParts = new ArrayList<>();
|
||||
partMatch = new TextPositionSequence(searchSpace.get(i).getPage());
|
||||
partMatch = new TextPositionSequence(sequences.get(i).getPage());
|
||||
}
|
||||
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0) {
|
||||
if(j == sequences.get(i).length() -1 && counter != 0){
|
||||
crossSequenceParts.add(partMatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return finalMatches;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts){
|
||||
|
||||
String plainId = IdBuilder.buildId(crossSequenceParts);
|
||||
String id = plainId;
|
||||
UUID id = UUID.randomUUID();
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
int currentPage = -1;
|
||||
int idDiffentPageSuffix = 1;
|
||||
EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
|
||||
for (TextPositionSequence textPositionSequence : crossSequenceParts) {
|
||||
if (currentPage == -1) {
|
||||
for (TextPositionSequence textPositionSequence :crossSequenceParts){
|
||||
if(currentPage == -1){
|
||||
currentPage = textPositionSequence.getPage();
|
||||
entityPositionSequence.setPageNumber(currentPage);
|
||||
entityPositionSequence.getSequences().add(textPositionSequence);
|
||||
} else if (currentPage == textPositionSequence.getPage()) {
|
||||
} else if(currentPage == textPositionSequence.getPage()){
|
||||
entityPositionSequence.getSequences().add(textPositionSequence);
|
||||
} else {
|
||||
id = plainId + "-" + idDiffentPageSuffix;
|
||||
idDiffentPageSuffix++;
|
||||
result.add(entityPositionSequence);
|
||||
entityPositionSequence = new EntityPositionSequence(id);
|
||||
entityPositionSequence.setPageNumber(textPositionSequence.getPage());
|
||||
entityPositionSequence.getSequences().add(textPositionSequence);
|
||||
currentPage = textPositionSequence.getPage();
|
||||
}
|
||||
}
|
||||
result.add(entityPositionSequence);
|
||||
@@ -165,21 +114,20 @@ public class SearchableText {
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
if (Math.abs(previous.getY1() - word.getY1()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
@@ -189,21 +137,17 @@ public class SearchableText {
|
||||
previous = word;
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" {2}", " ");
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" ", " ");
|
||||
}
|
||||
|
||||
|
||||
public String getAsStringWithLinebreaks() {
|
||||
|
||||
public String getAsStringWithLinebreaks(){
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
if (Math.abs(previous.getY1() - word.getY1()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
@@ -215,4 +159,4 @@ public class SearchableText {
|
||||
return sb.append("\n").toString();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+68
-331
@@ -1,38 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Data
|
||||
@Slf4j
|
||||
@Builder
|
||||
public class Section {
|
||||
|
||||
private boolean isLocal;
|
||||
|
||||
private Set<String> dictionaryTypes;
|
||||
|
||||
@Builder.Default
|
||||
private Map<String, Set<String>> localDictionaryAdds = new HashMap<>();
|
||||
|
||||
private Set<Entity> entities;
|
||||
|
||||
// This still contains linebreaks etc.
|
||||
@@ -41,362 +23,117 @@ public class Section {
|
||||
//This does not contain linebreaks and must always be used for correct offsets.
|
||||
private String searchText;
|
||||
|
||||
private String headline;
|
||||
|
||||
private int sectionNumber;
|
||||
|
||||
private Map<String, CellValue> tabularData;
|
||||
|
||||
private Dictionary dictionary;
|
||||
|
||||
private SearchableText searchableText;
|
||||
|
||||
|
||||
public boolean rowEquals(String headerName, String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
|
||||
.toString()
|
||||
.equals(value);
|
||||
}
|
||||
|
||||
|
||||
public boolean hasTableHeader(String headerName) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName);
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesType(String type) {
|
||||
|
||||
public boolean contains(String type) {
|
||||
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
public boolean headlineContainsWord(String word) {
|
||||
|
||||
return StringUtils.containsIgnoreCase(headline, word);
|
||||
}
|
||||
|
||||
|
||||
public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Set<Entity> expanded = new HashSet<>();
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
|
||||
.getRedactionReason(), entity.getLegalBasis()));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
|
||||
|
||||
public void redact(String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
public void redact(String type, int ruleNumber, String reason){
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
if(entity.getType().equals(type)){
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void redactNot(String type, int ruleNumber, String reason) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
public void redactNot(String type, int ruleNumber, String reason){
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
if(entity.getType().equals(type)){
|
||||
entity.setRedaction(false);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
public void highlightAll(String type){
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
|
||||
if(entity.getType().equals(type)){
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void addHintAnnotation(String value, String asType) {
|
||||
public void redactLineAfter(String start, String asType, int ruleNumber, String reason){
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
String value = StringUtils.substringBetween(text, start, "\n");
|
||||
|
||||
|
||||
public void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason,
|
||||
String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
if(value != null){
|
||||
Set<Entity> found = findEntity(value.trim(), asType);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void recommendLineAfter(String start, String asType) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
|
||||
String trimmedValue = value.trim();
|
||||
String cleanValue;
|
||||
if (trimmedValue.startsWith(":")) {
|
||||
cleanValue = trimmedValue.substring(1).trim();
|
||||
} else {
|
||||
cleanValue = trimmedValue;
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>())
|
||||
.add(cleanValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
|
||||
String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addRecommendationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
|
||||
String reason, String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(searchText, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
|
||||
String reason, String legalBasis) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, stop);
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String[] lines = value.split("\n");
|
||||
|
||||
for (String line : lines) {
|
||||
|
||||
if (line.trim().length() <= 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void highlightCell(String cellHeader, int ruleNumber, String type) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
|
||||
}
|
||||
|
||||
|
||||
public void redactCell(String cellHeader, int ruleNumber, String type, boolean addAsRecommendations, String reason,
|
||||
String legalBasis) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
|
||||
}
|
||||
|
||||
|
||||
public void redactNotCell(String cellHeader, int ruleNumber, String type, boolean addAsRecommendations,
|
||||
String reason) {
|
||||
|
||||
annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseInsensitive ? value.toLowerCase() : value;
|
||||
|
||||
Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true);
|
||||
|
||||
found.forEach(entity -> {
|
||||
if (redacted) {
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if(entity.getType().equals(asType)){
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
|
||||
}
|
||||
});
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
|
||||
boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
|
||||
String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason){
|
||||
|
||||
CellValue value = tabularData.get(cleanHeaderName);
|
||||
if (value == null) {
|
||||
log.warn("Could not find any data for {}.", cellHeader);
|
||||
} else {
|
||||
String word = value.toString();
|
||||
String value = StringUtils.substringBetween(searchText, start, stop);
|
||||
|
||||
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false);
|
||||
entity.setRedaction(redact);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setTargetSequences(value.getTextBlocks()
|
||||
.stream()
|
||||
.map(TextBlock::getSequences)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
|
||||
entity.setLegalBasis(legalBasis);
|
||||
if(value != null){
|
||||
Set<Entity> found = findEntity(value.trim(), asType);
|
||||
entities.addAll(found);
|
||||
}
|
||||
|
||||
Set<Entity> singleEntitySet = new HashSet<>();
|
||||
singleEntitySet.add(entity);
|
||||
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if(entity.getType().equals(asType)){
|
||||
entity.setRedaction(true);
|
||||
entity.setRedactionReason("\nRule " + ruleNumber + " matched\n\n" +reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
|
||||
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
|
||||
if (addAsRecommendations && !isLocal()) {
|
||||
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
|
||||
Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
|
||||
Matcher matcher = pattern.matcher(cleanedWord);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group().trim();
|
||||
if (match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>())
|
||||
.add(match);
|
||||
String lastname = match.split(" ")[0];
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + type, (x) -> new HashSet<>())
|
||||
.add(lastname);
|
||||
private Set<Entity> findEntity(String value, String asType) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = searchText.indexOf(value, stopIndex);
|
||||
stopIndex = startIndex + value.length();
|
||||
|
||||
if (startIndex > -1 &&
|
||||
(startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText.charAt(startIndex - 1))) &&
|
||||
(stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
|
||||
found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
|
||||
|
||||
removeEntitiesContainedInLarger(found);
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class SectionSearchableTextPair {
|
||||
|
||||
private Section section;
|
||||
private SearchableText searchableText;
|
||||
|
||||
}
|
||||
-268
@@ -1,268 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
|
||||
List<SectionRectangle> sectionRectangles = sectionGrid.getRectanglesPerPage().get(page);
|
||||
if (sectionRectangles != null && !sectionRectangles.isEmpty()) {
|
||||
drawSectionGrid(document, pdPage, sectionRectangles);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
|
||||
if (logEntries != null && !logEntries.isEmpty()) {
|
||||
addAnnotations(logEntries, pdPage, page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
|
||||
for (RedactionLogEntry entry : logEntries) {
|
||||
annotations.addAll(createAnnotation(entry, page, pdPage.getMediaBox(), pdPage.getCropBox()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, PDRectangle mediaBox,
|
||||
PDRectangle cropBox) {
|
||||
|
||||
List<PDAnnotation> annotations = new ArrayList<>();
|
||||
|
||||
List<Rectangle> rectangles = redactionLogEntry.getPositions()
|
||||
.stream()
|
||||
.filter(pos -> pos.getPage() == page)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (rectangles.isEmpty()) {
|
||||
return annotations;
|
||||
}
|
||||
|
||||
PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
|
||||
annotation.constructAppearances();
|
||||
PDRectangle pdRectangle = toPDRectangle(rectangles, mediaBox, cropBox);
|
||||
annotation.setRectangle(pdRectangle);
|
||||
annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
|
||||
if (!redactionLogEntry.isHint()) {
|
||||
annotation.setContents(createAnnotationContent(redactionLogEntry));
|
||||
}
|
||||
annotation.setTitlePopup(redactionLogEntry.getId());
|
||||
annotation.setAnnotationName(redactionLogEntry.getId());
|
||||
annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
|
||||
annotations.add(annotation);
|
||||
|
||||
if (redactionLogEntry.getComments() != null) {
|
||||
for (Comment comment : redactionLogEntry.getComments()) {
|
||||
PDAnnotationText txtAnnot = new PDAnnotationText();
|
||||
txtAnnot.setAnnotationName(comment.getId());
|
||||
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
|
||||
txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
|
||||
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));
|
||||
txtAnnot.setTitlePopup(comment.getUser());
|
||||
txtAnnot.setContents(comment.getText());
|
||||
txtAnnot.setRectangle(pdRectangle);
|
||||
annotations.add(txtAnnot);
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
|
||||
private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.isManual()) {
|
||||
return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
return "\nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
|
||||
.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
|
||||
|
||||
private PDRectangle toPDRectangle(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float lowerLeftX = Float.MAX_VALUE;
|
||||
float upperRightX = 0;
|
||||
float lowerLeftY = 0;
|
||||
float upperRightY = Float.MAX_VALUE;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
if (rectangle.getTopLeft().getX() < lowerLeftX) {
|
||||
lowerLeftX = rectangle.getTopLeft().getX();
|
||||
}
|
||||
if (rectangle.getTopLeft().getX() + rectangle.getWidth() > upperRightX) {
|
||||
upperRightX = rectangle.getTopLeft().getX() + rectangle.getWidth();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() + rectangle.getHeight() > lowerLeftY) {
|
||||
lowerLeftY = rectangle.getTopLeft().getY() + rectangle.getHeight();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() < upperRightY) {
|
||||
upperRightY = rectangle.getTopLeft().getY();
|
||||
}
|
||||
}
|
||||
|
||||
var x1 = lowerLeftX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = lowerLeftY + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = upperRightX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = upperRightY - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
PDRectangle annotationPosition = new PDRectangle();
|
||||
annotationPosition.setLowerLeftX(x1);
|
||||
annotationPosition.setLowerLeftY(y1);
|
||||
annotationPosition.setUpperRightX(x2);
|
||||
annotationPosition.setUpperRightY(y2);
|
||||
return annotationPosition;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoints(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float[] quadPoints = new float[rectangles.size() * 8];
|
||||
int i = 0;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
float[] quadPoint = toQuadPoint(rectangle, mediaBox, cropBox);
|
||||
for (int j = 0; j <= 7; j++) {
|
||||
quadPoints[i + j] = quadPoint[j];
|
||||
}
|
||||
i += 8;
|
||||
}
|
||||
return quadPoints;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoint(Rectangle rectangle, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
var x1 = rectangle.getTopLeft().getX() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = rectangle.getTopLeft().getY() + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = rectangle.getTopLeft()
|
||||
.getX() + rectangle.getWidth() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = rectangle.getTopLeft()
|
||||
.getY() + rectangle.getHeight() - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
|
||||
// of the area to be highlighted
|
||||
return new float[]{x1, y1, x2, y2, x1, y2 - rectangle.getHeight(), x2, y1 - rectangle.getHeight()};
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionGrid(PDDocument document, PDPage pdPage,
|
||||
List<SectionRectangle> sectionRectangles) throws IOException {
|
||||
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
for (SectionRectangle sectionRectangle : sectionRectangles) {
|
||||
drawSectionRectangle(contentStream, sectionRectangle);
|
||||
drawSectionPartNumberText(contentStream, sectionRectangle);
|
||||
drawTableCells(contentStream, sectionRectangle);
|
||||
}
|
||||
contentStream.close();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionRectangle(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.LIGHT_GRAY);
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.addRect(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY(), sectionRectangle.getWidth(), sectionRectangle.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionPartNumberText(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.beginText();
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
if (sectionRectangle.getTableCells() == null) {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY() + sectionRectangle.getHeight());
|
||||
} else {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY());
|
||||
}
|
||||
contentStream.showText(sectionRectangle.getPart() + "/" + sectionRectangle.getNumberOfParts());
|
||||
contentStream.endText();
|
||||
}
|
||||
|
||||
|
||||
private void drawTableCells(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
if (sectionRectangle.getTableCells() != null) {
|
||||
for (CellRectangle cell : sectionRectangle.getTableCells()) {
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.setStrokingColor(Color.CYAN);
|
||||
contentStream.addRect(cell.getTopLeft().getX(), cell.getTopLeft()
|
||||
.getY(), cell.getWidth(), cell.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Map<Integer, List<RedactionLogEntry>> convertRedactionLog(RedactionLog redactionLog) {
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = new HashMap<>();
|
||||
if (redactionLog == null) {
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
int page = 0;
|
||||
for (Rectangle position : entry.getPositions()) {
|
||||
if (position.getPage() != page) {
|
||||
redactionLogPerPage.computeIfAbsent(position.getPage(), x -> new ArrayList<>()).add(entry);
|
||||
page = position.getPage();
|
||||
}
|
||||
}
|
||||
}
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
|
||||
}
|
||||
+6
-174
@@ -1,184 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
|
||||
import feign.FeignException;
|
||||
import org.springframework.stereotype.Service;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DictionaryService {
|
||||
|
||||
|
||||
private final DictionaryClient dictionaryClient;
|
||||
|
||||
|
||||
private Map<String, DictionaryRepresentation> dictionariesByRuleSets = new HashMap<>();
|
||||
|
||||
|
||||
public void updateDictionary(String ruleSetId) {
|
||||
|
||||
long version = dictionaryClient.getVersion(ruleSetId);
|
||||
|
||||
var foundDictionary = dictionariesByRuleSets.get(ruleSetId);
|
||||
|
||||
if (foundDictionary == null || version > foundDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(ruleSetId, version);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void updateDictionaryEntry(String ruleSetId, long version) {
|
||||
|
||||
try {
|
||||
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
|
||||
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes(ruleSetId);
|
||||
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
|
||||
|
||||
List<DictionaryModel> dictionary = typeResponse.getTypes()
|
||||
.stream()
|
||||
.map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
|
||||
.isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
|
||||
dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
|
||||
|
||||
Colors colors = dictionaryClient.getColors(ruleSetId);
|
||||
|
||||
dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
|
||||
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
|
||||
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
|
||||
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
|
||||
dictionaryRepresentation.setRuleSetId(ruleSetId);
|
||||
dictionaryRepresentation.setDictionaryVersion(version);
|
||||
dictionaryRepresentation.setDictionary(dictionary);
|
||||
|
||||
dictionariesByRuleSets.put(ruleSetId, dictionaryRepresentation);
|
||||
}
|
||||
} catch (FeignException e) {
|
||||
log.warn("Got some unknown feignException", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void updateExternalDictionary(Dictionary dictionary, String ruleSetId) {
|
||||
dictionary.getDictionaryModels().forEach(dm -> {
|
||||
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
|
||||
dictionaryClient.addEntries(dm.getType(), ruleSetId, new ArrayList<>(dm.getLocalEntries()), false);
|
||||
long externalVersion = dictionaryClient.getVersion(ruleSetId);
|
||||
if (externalVersion == dictionary.getVersion() + 1) {
|
||||
dictionary.setVersion(externalVersion);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private Set<String> convertEntries(TypeResult t) {
|
||||
|
||||
if (t.isCaseInsensitive()) {
|
||||
return dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId())
|
||||
.getEntries()
|
||||
.stream()
|
||||
.map(String::toLowerCase)
|
||||
.collect(Collectors.toSet());
|
||||
} else {
|
||||
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getRuleSetId()).getEntries());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private float[] convertColor(String hex) {
|
||||
|
||||
Color color = Color.decode(hex);
|
||||
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
|
||||
}
|
||||
|
||||
|
||||
public boolean isCaseInsensitiveDictionary(String type, String ruleSetId) {
|
||||
|
||||
DictionaryModel dictionaryModel = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
if (dictionaryModel != null) {
|
||||
return dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public float[] getColor(String type, String ruleSetId) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
if (model != null) {
|
||||
return model.getColor();
|
||||
}
|
||||
return dictionariesByRuleSets.get(ruleSetId).getDefaultColor();
|
||||
}
|
||||
|
||||
|
||||
public boolean isHint(String type, String ruleSetId) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
if (model != null) {
|
||||
return model.isHint();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isRecommendation(String type, String ruleSetId) {
|
||||
|
||||
DictionaryModel model = dictionariesByRuleSets.get(ruleSetId).getLocalAccessMap().get(type);
|
||||
if (model != null) {
|
||||
return model.isRecommendation();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public Dictionary getDeepCopyDictionary(String ruleSetId) {
|
||||
List<DictionaryModel> copy = new ArrayList<>();
|
||||
|
||||
var representation = dictionariesByRuleSets.get(ruleSetId);
|
||||
var dictionary = dictionariesByRuleSets.get(ruleSetId).getDictionary();
|
||||
dictionary.forEach(dm -> {
|
||||
copy.add(SerializationUtils.clone(dm));
|
||||
});
|
||||
|
||||
return new Dictionary(copy, representation.getDictionaryVersion());
|
||||
}
|
||||
|
||||
public float[] getRequestRemoveColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
|
||||
}
|
||||
|
||||
public float[] getNotRedactedColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getNotRedactedColor();
|
||||
}
|
||||
|
||||
public float[] getRequestAddColor(String ruleSetId) {
|
||||
return dictionariesByRuleSets.get(ruleSetId).getRequestAddColor();
|
||||
}
|
||||
}
|
||||
public static final String VERTEBRATES_CODE = "vertebrate";
|
||||
public static final String ADDRESS_CODE = "address";
|
||||
public static final String NAME_CODE = "name";
|
||||
public static final String NO_REDACTION_INDICATOR = "default";
|
||||
}
|
||||
|
||||
+24
-90
@@ -3,8 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
@@ -13,124 +13,58 @@ import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DroolsExecutionService {
|
||||
|
||||
private final RulesClient rulesClient;
|
||||
@Autowired
|
||||
private KieContainer kieContainer;
|
||||
|
||||
private Map<String, KieContainer> kieContainers = new HashMap<>();
|
||||
private String currentDrlRules;
|
||||
|
||||
private Map<String, Long> rulesVersionPerRuleSetId = new HashMap<>();
|
||||
|
||||
|
||||
public KieContainer getKieContainer(String ruleSetId) {
|
||||
|
||||
KieContainer container = kieContainers.get(ruleSetId);
|
||||
if (container == null) {
|
||||
return createOrUpdateKieContainer(ruleSetId);
|
||||
} else {
|
||||
return container;
|
||||
}
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
currentDrlRules = ResourceLoader.loadAsString("drools/rules.drl");
|
||||
}
|
||||
|
||||
|
||||
public Section executeRules(KieContainer kieContainer, Section section) {
|
||||
|
||||
public Section executeRules(Section section) {
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("section", section);
|
||||
kieSession.insert(section);
|
||||
kieSession.fireAllRules();
|
||||
kieSession.dispose();
|
||||
|
||||
return section;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public KieContainer updateRules(String ruleSetId) {
|
||||
|
||||
long version = rulesClient.getVersion(ruleSetId);
|
||||
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
|
||||
if (rulesVersion == null) {
|
||||
rulesVersion = -1L;
|
||||
}
|
||||
|
||||
if (version > rulesVersion.longValue()) {
|
||||
rulesVersionPerRuleSetId.put(ruleSetId, version);
|
||||
return createOrUpdateKieContainer(ruleSetId);
|
||||
}
|
||||
return getKieContainer(ruleSetId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieContainer createOrUpdateKieContainer(String ruleSetId) {
|
||||
public void updateRules(String drlAsString) {
|
||||
|
||||
try {
|
||||
|
||||
RulesResponse rules = rulesClient.getRules(ruleSetId);
|
||||
if (rules == null || StringUtils.isEmpty(rules.getRules())) {
|
||||
if (StringUtils.isEmpty(drlAsString)) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule(ruleSetId, rules.getRules(), kieServices);
|
||||
|
||||
var container = kieContainers.get(ruleSetId);
|
||||
if (container != null) {
|
||||
container.updateToVersion(kieModule.getReleaseId());
|
||||
return container;
|
||||
}
|
||||
|
||||
container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
kieContainers.put(ruleSetId, container);
|
||||
return container;
|
||||
InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
kieContainer.updateToVersion(kieModule.getReleaseId());
|
||||
currentDrlRules = drlAsString;
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
throw new RulesValidationException("Could not update rules", e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieModule getKieModule(String ruleSetId, String rules, KieServices kieServices) {
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules" + ruleSetId + ".drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
return kieBuilder.getKieModule();
|
||||
}
|
||||
|
||||
|
||||
public void testRules(String rules) {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule("test-rules", rules, kieServices);
|
||||
var container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
container.newKieSession();
|
||||
container.dispose();
|
||||
}
|
||||
|
||||
|
||||
public long getRulesVersion(String ruleSetId) {
|
||||
|
||||
Long rulesVersion = rulesVersionPerRuleSetId.get(ruleSetId);
|
||||
if (rulesVersion == null) {
|
||||
return -1;
|
||||
}
|
||||
return rulesVersion.longValue();
|
||||
public String getRules() {
|
||||
return currentDrlRules;
|
||||
}
|
||||
|
||||
}
|
||||
+101
-262
@@ -1,321 +1,160 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class EntityRedactionService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
private final DictionaryClient dictionaryClient;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final SurroundingWordsService surroundingWordsService;
|
||||
|
||||
|
||||
public void processDocument(Document classifiedDoc, String ruleSetId, ManualRedactions manualRedactions) {
|
||||
|
||||
dictionaryService.updateDictionary(ruleSetId);
|
||||
KieContainer container = droolsExecutionService.updateRules(ruleSetId);
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(ruleSetId);
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(ruleSetId);
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
|
||||
|
||||
if (dictionary.hasLocalEntries()) {
|
||||
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
|
||||
documentEntities.stream().forEach(entity -> {
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
|
||||
.add(entity);
|
||||
}
|
||||
});
|
||||
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
|
||||
}
|
||||
|
||||
for (Entity entity : documentEntities) {
|
||||
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
|
||||
.add(entityPositionSequence);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
|
||||
classifiedDoc.getEntities()
|
||||
.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
|
||||
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
|
||||
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
|
||||
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter()));
|
||||
}
|
||||
}
|
||||
|
||||
dictionaryService.updateExternalDictionary(dictionary, ruleSetId);
|
||||
|
||||
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
|
||||
classifiedDoc.setRulesVersion(rulesVersion);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(Document classifiedDoc, KieContainer kieContainer,
|
||||
ManualRedactions manualRedactions, Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
public void processDocument(Document classifiedDoc) {
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>();
|
||||
|
||||
AtomicInteger sectionNumber = new AtomicInteger(1);
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
SearchableText searchableText = paragraph.getSearchableText();
|
||||
|
||||
List<Table> tables = paragraph.getTables();
|
||||
|
||||
List<SearchableText> searchableRows = new ArrayList<>();
|
||||
for (Table table : tables) {
|
||||
if (table.getColCount() == 2) {
|
||||
sectionSearchableTextPairs.addAll(processTableAsOneText(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
} else {
|
||||
sectionSearchableTextPairs.addAll(processTablePerRow(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
for (Cell column : row) {
|
||||
if (column == null || column.getTextBlocks() == null) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : column.getTextBlocks()) {
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
}
|
||||
searchableRows.add(searchableRow);
|
||||
}
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
sectionSearchableTextPairs.add(processText(paragraph, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
sectionNumber.incrementAndGet();
|
||||
|
||||
Set<Entity> entities = findEntities(searchableText);
|
||||
Section analysedSection = droolsExecutionService.executeRules(Section
|
||||
.builder()
|
||||
.entities(entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.build());
|
||||
|
||||
for (Entity entity : analysedSection.getEntities()) {
|
||||
entity.setPositionSequences(searchableText.getSequences(entity.getWord()));
|
||||
}
|
||||
|
||||
documentEntities.addAll(analysedSection.getEntities());
|
||||
|
||||
for (SearchableText searchableRow : searchableRows) {
|
||||
Set<Entity> rowEntities = findEntities(searchableRow);
|
||||
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(Section
|
||||
.builder()
|
||||
.entities(rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.build());
|
||||
|
||||
for (Entity entity : analysedRowSection.getEntities()) {
|
||||
entity.setPositionSequences(searchableRow.getSequences(entity.getWord()));
|
||||
}
|
||||
documentEntities.addAll(analysedRowSection.getEntities());
|
||||
}
|
||||
}
|
||||
|
||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
||||
documentEntities.addAll(analysedRowSection.getEntities());
|
||||
|
||||
analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
|
||||
if (dictionary.isRecommendation(key)) {
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
if (!dictionary.containsValue(key, value)) {
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key) == null) {
|
||||
log.warn("Dictionary {} is null", key);
|
||||
}
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
|
||||
log.warn("Dictionary {} localEntries is null", key);
|
||||
}
|
||||
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
});
|
||||
}
|
||||
documentEntities.forEach(entity -> {
|
||||
entity.getPositionSequences().forEach(sequence -> {
|
||||
classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
|
||||
new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence))
|
||||
);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
return documentEntities;
|
||||
}
|
||||
|
||||
private Set<Entity> findEntities(SearchableText searchableText) {
|
||||
|
||||
private List<SectionSearchableTextPair> processTablePerRow(Table table, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
|
||||
int rowNumber = 0;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
Map<String, CellValue> tabularData = new HashMap<>();
|
||||
int start = 0;
|
||||
List<Integer> cellStarts = new ArrayList<>();
|
||||
for (Cell cell : row) {
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
int cellStart = start;
|
||||
|
||||
if(rowNumber != 0) {
|
||||
cell.getHeaderCells().forEach(headerCell -> {
|
||||
StringBuilder headerBuilder = new StringBuilder();
|
||||
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
|
||||
String headerName = headerBuilder.toString().replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
|
||||
});
|
||||
}
|
||||
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
// TODO avoid cell overlap merging.
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
cellStarts.add(cellStart);
|
||||
start = start + cell.toString().trim().length() + 1;
|
||||
}
|
||||
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.tabularData(tabularData)
|
||||
.searchableText(searchableRow)
|
||||
.dictionary(dictionary)
|
||||
.build(), searchableRow));
|
||||
|
||||
sectionNumber.incrementAndGet();
|
||||
rowNumber++;
|
||||
}
|
||||
return sectionSearchableTextPairs;
|
||||
}
|
||||
|
||||
|
||||
private List<SectionSearchableTextPair> processTableAsOneText(Table table, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
SearchableText entireTableText = new SearchableText();
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
entireTableText.addAll(textBlock.getSequences());
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
}
|
||||
}
|
||||
|
||||
Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(entireTableText.getAsStringWithLinebreaks())
|
||||
.searchText(entireTableText.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(entireTableText)
|
||||
.dictionary(dictionary)
|
||||
.build(), entireTableText));
|
||||
|
||||
return sectionSearchableTextPairs;
|
||||
}
|
||||
|
||||
|
||||
private SectionSearchableTextPair processText(Paragraph paragraph, ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
SearchableText searchableText = paragraph.getSearchableText();
|
||||
addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber
|
||||
.intValue());
|
||||
Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
|
||||
|
||||
return new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(paragraph.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(searchableText)
|
||||
.dictionary(dictionary)
|
||||
.build(), searchableText);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local) {
|
||||
String normalizedInputString = searchableText.toString();
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
String searchableString = searchableText.toString();
|
||||
if (StringUtils.isEmpty(searchableString)) {
|
||||
return found;
|
||||
}
|
||||
for (Map.Entry<String, Set<String>> entry : getDictionaryEntry().entrySet()) {
|
||||
for (String value : entry.getValue()) {
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = normalizedInputString.indexOf(value, stopIndex);
|
||||
stopIndex = startIndex + value.length();
|
||||
|
||||
String lowercaseInputString = searchableString.toLowerCase();
|
||||
for (DictionaryModel model : dictionary.getDictionaryModels()) {
|
||||
if (model.isCaseInsensitive()) {
|
||||
found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local));
|
||||
} else {
|
||||
found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local));
|
||||
if (startIndex > -1 &&
|
||||
(startIndex == 0 || Character.isWhitespace(normalizedInputString.charAt(startIndex - 1)) || isSeparator(normalizedInputString.charAt(startIndex - 1))) &&
|
||||
(stopIndex == normalizedInputString.length() || isSeparator(normalizedInputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
}
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
removeEntitiesContainedInLarger(found);
|
||||
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private void addSectionToManualRedactions(List<TextBlock> textBlocks, ManualRedactions manualRedactions,
|
||||
String section, int sectionNumber) {
|
||||
|
||||
if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) {
|
||||
return;
|
||||
private Map<String, Set<String>> getDictionaryEntry() {
|
||||
try {
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes();
|
||||
if (typeResponse == null || CollectionUtils.isEmpty(typeResponse.getTypes())) {
|
||||
return Collections.emptyMap();
|
||||
} else {
|
||||
List<String> types = typeResponse.getTypes().stream().map(typeResult -> typeResult.getType()).collect(Collectors.toList());
|
||||
return types.stream().filter(s -> !s.equals("default")).collect(Collectors.toMap(s -> s, s -> dictionaryClient.getDictionaryForType(s).getEntries().stream().collect(Collectors.toSet())));
|
||||
}
|
||||
} catch (FeignException e) {
|
||||
log.warn("Got some unknown feignException", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
for (TextBlock textBlock : textBlocks) {
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
|
||||
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
|
||||
if (textBlock.contains(rectangle)) {
|
||||
manualRedactionEntry.setSection(section);
|
||||
manualRedactionEntry.setSectionNumber(sectionNumber);
|
||||
}
|
||||
private boolean isSeparator(char c) {
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-368
@@ -1,368 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.IdRemoval;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Status;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogCreatorService {
|
||||
|
||||
private static final String IMAGE = "image";
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
|
||||
String ruleSetId) {
|
||||
|
||||
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
|
||||
|
||||
for (int page = 1; page <= numberOfPages; page++) {
|
||||
|
||||
addSectionGrid(classifiedDoc, page);
|
||||
|
||||
if (classifiedDoc.getEntities().get(page) != null) {
|
||||
addEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
if (manualRedactionPages.contains(page)) {
|
||||
addManualEntries(classifiedDoc, manualRedactions, page, ruleSetId);
|
||||
}
|
||||
|
||||
if (!classifiedDoc.getPages().get(page - 1).getImageBounds().isEmpty()) {
|
||||
addImageEntries(classifiedDoc, page, ruleSetId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addImageEntries(Document classifiedDoc, int pageNumber, String ruleSetId) {
|
||||
|
||||
for (Rectangle2D imageBounds : classifiedDoc.getPages().get(pageNumber - 1).getImageBounds()) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(IdBuilder.buildId(imageBounds, pageNumber))
|
||||
.color(getColor(IMAGE, ruleSetId))
|
||||
.type(IMAGE)
|
||||
.redacted(false)
|
||||
.isHint(true)
|
||||
.manual(false)
|
||||
.isDictionaryEntry(false)
|
||||
.isRecommendation(false)
|
||||
.positions(List.of(new Rectangle(new Point((float) imageBounds.getX(), (float) imageBounds.getY()), (float) imageBounds
|
||||
.getWidth(), (float) imageBounds.getHeight(), pageNumber)))
|
||||
.build();
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
|
||||
|
||||
Set<Integer> manualRedactionPages = new HashSet<>();
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
manualRedactions.getEntriesToAdd().forEach(entry -> {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
manualRedactionPages.add(pos.getPage());
|
||||
});
|
||||
});
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
|
||||
private void addEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page, String ruleSetId) {
|
||||
|
||||
// Duplicates can exist due table extraction colums over multiple rows.
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
|
||||
entityLoop:
|
||||
for (Entity entity : classifiedDoc.getEntities().get(page)) {
|
||||
|
||||
List<Comment> comments = null;
|
||||
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, ruleSetId);
|
||||
if (processedIds.contains(entityPositionSequence.getId())) {
|
||||
|
||||
// TODO refactor this outer loop jump as soon as we have the time.
|
||||
continue entityLoop;
|
||||
} else {
|
||||
processedIds.add(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
|
||||
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
|
||||
if (manualRemoval.getId().equals(entityPositionSequence.getId())) {
|
||||
comments = manualRedactions.getComments().get(manualRemoval.getId());
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
|
||||
entity.setRedaction(false);
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
|
||||
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
|
||||
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
|
||||
.stream()
|
||||
.flatMap(seq -> seq.getTextPositions().stream())
|
||||
.collect(Collectors.toList()), page);
|
||||
|
||||
if (manualRedactions != null) {
|
||||
comments = manualRedactions.getComments().get(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(comments);
|
||||
redactionLogEntry.getPositions().addAll(rectanglesPerLine);
|
||||
|
||||
}
|
||||
|
||||
redactionLogEntry.setId(entityPositionSequence.getId());
|
||||
|
||||
// FIXME ids should never be null. Figure out why this happens.
|
||||
if (redactionLogEntry.getId() != null) {
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> getRectanglesPerLine(List<TextPosition> textPositions, int page) {
|
||||
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
if (textPositions.size() == 1) {
|
||||
rectangles.add(new TextPositionSequence(textPositions, page).getRectangle());
|
||||
} else {
|
||||
float y = textPositions.get(0).getYDirAdj();
|
||||
int startIndex = 0;
|
||||
for (int i = 1; i < textPositions.size(); i++) {
|
||||
float yDirAdj = textPositions.get(i).getYDirAdj();
|
||||
if (yDirAdj != y) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, i), page).getRectangle());
|
||||
y = yDirAdj;
|
||||
startIndex = i;
|
||||
}
|
||||
}
|
||||
if (startIndex != textPositions.size() - 1) {
|
||||
rectangles.add(new TextPositionSequence(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
|
||||
}
|
||||
}
|
||||
|
||||
return rectangles;
|
||||
}
|
||||
|
||||
|
||||
private void addManualEntries(Document classifiedDoc, ManualRedactions manualRedactions, int page,
|
||||
String ruleSetId) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
|
||||
|
||||
String id = manualRedactionEntry.getId();
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, ruleSetId);
|
||||
|
||||
List<Rectangle> rectanglesOnPage = new ArrayList<>();
|
||||
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
|
||||
if (page == rectangle.getPage()) {
|
||||
rectanglesOnPage.add(rectangle);
|
||||
redactionLogEntry.getPositions().add(rectangle);
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(manualRedactions.getComments().get(id));
|
||||
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
|
||||
classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
|
||||
String ruleSetId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), ruleSetId, manualRedactionEntry.getStatus()))
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.sectionNumber(manualRedactionEntry.getSectionNumber())
|
||||
.manual(true)
|
||||
.status(manualRedactionEntry.getStatus())
|
||||
.manualRedactionType(ManualRedactionType.ADD)
|
||||
.isDictionaryEntry(false)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(Entity entity, String ruleSetId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.color(getColor(entity, ruleSetId, false))
|
||||
.reason(entity.getRedactionReason())
|
||||
.legalBasis(entity.getLegalBasis())
|
||||
.value(entity.getWord())
|
||||
.type(entity.getType())
|
||||
.redacted(entity.isRedaction())
|
||||
.isHint(isHint(entity, ruleSetId))
|
||||
.isRecommendation(isRecommendation(entity, ruleSetId))
|
||||
.section(entity.getHeadline())
|
||||
.sectionNumber(entity.getSectionNumber())
|
||||
.matchedRule(entity.getMatchedRule())
|
||||
.isDictionaryEntry(entity.isDictionaryEntry())
|
||||
.textAfter(entity.getTextAfter())
|
||||
.textBefore(entity.getTextBefore())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(Entity entity, String ruleSetId, boolean requestedToRemove) {
|
||||
|
||||
if (requestedToRemove) {
|
||||
return dictionaryService.getRequestRemoveColor(ruleSetId);
|
||||
}
|
||||
if (!entity.isRedaction() && !isHint(entity, ruleSetId)) {
|
||||
return dictionaryService.getNotRedactedColor(ruleSetId);
|
||||
}
|
||||
return dictionaryService.getColor(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForManualAdd(String type, String ruleSetId, Status status) {
|
||||
|
||||
if (status.equals(Status.REQUESTED)) {
|
||||
return dictionaryService.getRequestAddColor(ruleSetId);
|
||||
} else if (status.equals(Status.DECLINED)) {
|
||||
return dictionaryService.getNotRedactedColor(ruleSetId);
|
||||
}
|
||||
return getColor(type, ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String ruleSetId) {
|
||||
|
||||
return dictionaryService.getColor(type, ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isHint(Entity entity, String ruleSetId) {
|
||||
|
||||
return dictionaryService.isHint(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isRecommendation(Entity entity, String ruleSetId) {
|
||||
|
||||
return dictionaryService.isRecommendation(entity.getType(), ruleSetId);
|
||||
}
|
||||
|
||||
|
||||
private void addSectionGrid(Document classifiedDoc, int page) {
|
||||
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
|
||||
AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
|
||||
|
||||
if (textBlock.getPage() != page) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (textBlock instanceof TextBlock) {
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size()));
|
||||
|
||||
} else if (textBlock instanceof Table) {
|
||||
|
||||
List<CellRectangle> cellRectangles = new ArrayList<>();
|
||||
for (List<Cell> row : ((Table) textBlock).getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (cell != null) {
|
||||
cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size(), cellRectangles));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-140
@@ -1,140 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SurroundingWordsService {
|
||||
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
|
||||
List<Integer> cellstarts) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String searchableString = searchableText.toString();
|
||||
|
||||
if (cellstarts != null) {
|
||||
for (int i = 0; i < cellstarts.size(); i++) {
|
||||
|
||||
int startOffset = cellstarts.get(i);
|
||||
int endOffset = -1;
|
||||
|
||||
if (i + 1 < cellstarts.size()) {
|
||||
endOffset = cellstarts.get(i + 1);
|
||||
} else {
|
||||
endOffset = searchableString.length() - 1;
|
||||
}
|
||||
|
||||
String text = searchableString.substring(startOffset, endOffset);
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
|
||||
int entityStartOffset = entity.getStart() - startOffset;
|
||||
int entityEndOffset = entity.getEnd() - startOffset;
|
||||
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
|
||||
|
||||
int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
|
||||
.getSurroundingWordsOffsetWindow();
|
||||
String textBefore = text.substring(offsetBefore, entityStartOffset);
|
||||
if (!textBefore.isBlank()) {
|
||||
String[] wordsBefore = textBefore.split(" ");
|
||||
int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsBefore.length;
|
||||
if (wordsBefore.length > 0) {
|
||||
entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
|
||||
}
|
||||
}
|
||||
|
||||
int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
|
||||
.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
|
||||
String textAfter = text.substring(entityEndOffset, endOffset);
|
||||
if (!textAfter.isBlank()) {
|
||||
String[] wordsAfter = textAfter.split(" ");
|
||||
int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsAfter.length;
|
||||
if (wordsAfter.length > 0) {
|
||||
entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsBefore(String[] words, int number, boolean endWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int startNumber = words.length > number ? words.length - number : 0;
|
||||
|
||||
for (int i = startNumber; i < words.length; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return endWithSpace ? result + " " : result;
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsAfter(String[] words, int number, boolean startWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < number; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return startWithSpace ? " " + result : result;
|
||||
}
|
||||
|
||||
}
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@UtilityClass
|
||||
public class EntitySearchUtils {
|
||||
|
||||
public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
|
||||
boolean local) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
for (String value : values) {
|
||||
|
||||
String cleanValue = value.trim();
|
||||
|
||||
if (cleanValue.length() <= 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(cleanValue, stopIndex);
|
||||
stopIndex = startIndex + cleanValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
|
||||
|
||||
Map<String, List<Entity>> entitiesByWord = new HashMap<>();
|
||||
|
||||
for (Entity entity : entities) {
|
||||
entitiesByWord.computeIfAbsent(entity.getWord(), (x) -> new ArrayList<>()).add(entity);
|
||||
}
|
||||
|
||||
for (String word : entitiesByWord.keySet()) {
|
||||
|
||||
List<Entity> orderedEntities = entitiesByWord.get(word)
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
Entity firstEntity = orderedEntities.get(0);
|
||||
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity
|
||||
.getType()), firstEntity.getTargetSequences());
|
||||
|
||||
for (int i = 0; i <= orderedEntities.size() - 1; i++) {
|
||||
try {
|
||||
orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
|
||||
} catch (Exception e) {
|
||||
log.warn("Mismatch between EntityPositionSequence and found Entity!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
removeEntitiesContainedInLarger(entities);
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
|
||||
.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Set<Entity> found, Dictionary dictionary) {
|
||||
found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary));
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Entity found, Dictionary dictionary) {
|
||||
|
||||
if(entities.contains(found)){
|
||||
Entity existing = entities.stream().filter(entity -> entity.equals(found)).findFirst().get();
|
||||
if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())){
|
||||
entities.remove(found);
|
||||
}
|
||||
}
|
||||
entities.add(found);
|
||||
}
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class IdBuilder {
|
||||
|
||||
private final HashFunction hashFunction = Hashing.murmur3_128();
|
||||
|
||||
public String buildId(List<TextPositionSequence> crossSequenceParts) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
crossSequenceParts.forEach(sequencePart -> sequencePart.getTextPositions().forEach(textPosition -> {
|
||||
sb.append(textPosition.getTextMatrix()).append(sequencePart.getPage());
|
||||
}));
|
||||
|
||||
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
public String buildId(Rectangle2D rectangle2D, int page){
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("x").append(rectangle2D.getX()).append("y").append(rectangle2D.getY()).append("h").append(rectangle2D.getHeight()).append("w").append(rectangle2D.getWidth()).append("p").append(page);
|
||||
|
||||
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@UtilityClass
|
||||
public class Patterns {
|
||||
|
||||
public static Map<String, Pattern> patternCache = new HashMap<>();
|
||||
|
||||
public static Pattern AUTHOR_TABLE_SPITTER = Pattern.compile("((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2}\\.)+|((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2} )+");
|
||||
|
||||
|
||||
public Pattern getCompiledPattern(String pattern, boolean caseInsensitive) {
|
||||
|
||||
String patternKey = pattern + caseInsensitive;
|
||||
if (patternCache.containsKey(patternKey)) {
|
||||
return patternCache.get(patternKey);
|
||||
}
|
||||
Pattern compiledPattern = Pattern.compile(pattern, caseInsensitive ? Pattern.CASE_INSENSITIVE : 0);
|
||||
patternCache.put(patternKey, compiledPattern);
|
||||
return compiledPattern;
|
||||
}
|
||||
|
||||
}
|
||||
+30
-3
@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -19,12 +20,38 @@ public class ResourceLoader {
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
try (InputStream is = resource.openStream();
|
||||
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
|
||||
BufferedReader br = new BufferedReader(isr)) {
|
||||
return br.lines().collect(Collectors.toSet());
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public String loadAsString(String classpathPath) {
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
|
||||
}
|
||||
try (InputStream is = resource.openStream();
|
||||
InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8);
|
||||
BufferedReader br = new BufferedReader(isr)) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ public class TextNormalizationUtilities {
|
||||
* @return Text without line-break hyphenation.
|
||||
*/
|
||||
public static String removeHyphenLineBreaks(String text) {
|
||||
return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
|
||||
return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
|
||||
}
|
||||
|
||||
}
|
||||
+9
-14
@@ -21,6 +21,7 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -28,6 +29,7 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@SuppressWarnings("PMD")
|
||||
public class PdfSegmentationService {
|
||||
|
||||
private final RulingCleaningService rulingCleaningService;
|
||||
@@ -36,7 +38,6 @@ public class PdfSegmentationService {
|
||||
private final ClassificationService classificationService;
|
||||
private final SectionsBuilderService sectionsBuilderService;
|
||||
|
||||
|
||||
public Document parseDocument(PDDocument pdDocument) throws IOException {
|
||||
|
||||
Document document = new Document();
|
||||
@@ -48,7 +49,6 @@ public class PdfSegmentationService {
|
||||
stripper.setPageNumber(pageNumber);
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
stripper.setPdpage(pdPage);
|
||||
stripper.getText(pdDocument);
|
||||
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
@@ -57,20 +57,19 @@ public class PdfSegmentationService {
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isRotated = rotation != 0 && rotation != 360;
|
||||
|
||||
ParsedElements parsedElements = ParsedElements.builder()
|
||||
ParsedElements parsedElements = ParsedElements
|
||||
.builder()
|
||||
.rulings(stripper.getRulings())
|
||||
.sequences(stripper.getTextPositionSequences())
|
||||
.imageBounds(stripper.getImageBounds())
|
||||
.minCharWidth(stripper.getMinCharWidths())
|
||||
.minCharWidth(Utils.round(stripper.getMinCharWidth(), 2))
|
||||
.minCharHeight(Utils.round(stripper.getMinCharHeight(), 2))
|
||||
.landscape(isLandscape)
|
||||
.rotated(isRotated)
|
||||
.build();
|
||||
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
|
||||
.getMinCharWidth());
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements.getMinCharWidth(), parsedElements.getMinCharHeight());
|
||||
|
||||
Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
|
||||
.getVertical());
|
||||
Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
|
||||
page.setRotation(rotation);
|
||||
|
||||
tableExtractionService.extractTables(cleanRulings, page);
|
||||
@@ -81,10 +80,8 @@ public class PdfSegmentationService {
|
||||
|
||||
page.setPageNumber(pageNumber);
|
||||
increaseDocumentStatistics(page, document);
|
||||
page.setImageBounds(parsedElements.getImageBounds());
|
||||
pages.add(page);
|
||||
}
|
||||
|
||||
document.setPages(pages);
|
||||
|
||||
classificationService.classifyDocument(document);
|
||||
@@ -92,11 +89,10 @@ public class PdfSegmentationService {
|
||||
sectionsBuilderService.buildSections(document);
|
||||
|
||||
return document;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void increaseDocumentStatistics(Page page, Document document) {
|
||||
|
||||
if (!page.isLandscape()) {
|
||||
document.getFontSizeCounter().addAll(page.getFontSizeCounter().getCountPerValue());
|
||||
}
|
||||
@@ -105,7 +101,6 @@ public class PdfSegmentationService {
|
||||
document.getFontStyleCounter().addAll(page.getFontStyleCounter().getCountPerValue());
|
||||
}
|
||||
|
||||
|
||||
private void buildPageStatistics(Page page) {
|
||||
|
||||
// Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
|
||||
|
||||
+26
-118
@@ -1,12 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
@@ -14,94 +11,53 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("all")
|
||||
public class SectionsBuilderService {
|
||||
|
||||
|
||||
public void buildSections(Document document) {
|
||||
|
||||
List<AbstractTextContainer> chunkWords = new ArrayList<>();
|
||||
List<Paragraph> chunkBlockList = new ArrayList<>();
|
||||
List<Paragraph> chunkBlockList1 = new ArrayList<>();
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
|
||||
String lastHeadline = "";
|
||||
Table previousTable = null;
|
||||
for (Page page : document.getPages()) {
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
|
||||
if (current.getClassification() == null || current.getClassification()
|
||||
.equals("Header") || current.getClassification().equals("Footer")) {
|
||||
if (current.getClassification() == null || current.getClassification().equals("Header") || current.getClassification().equals("Footer")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
current.setPage(page.getPageNumber());
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if(document.isHeadlines()) {
|
||||
lastHeadline = current.getText();
|
||||
}
|
||||
chunkBlockList.add(chunkBlock);
|
||||
if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
|
||||
Paragraph cb1 = buildTextBlock(chunkWords);
|
||||
chunkBlockList1.add(cb1);
|
||||
chunkWords = new ArrayList<>();
|
||||
if (CollectionUtils.isNotEmpty(chunkBlock.getTables())) {
|
||||
previousTable = chunkBlock.getTables().get(chunkBlock.getTables().size() - 1);
|
||||
}
|
||||
}
|
||||
if (current instanceof Table) {
|
||||
Table table = (Table) current;
|
||||
// Distribute header information for subsequent tables
|
||||
mergeTableMetadata(table, previousTable);
|
||||
previousTable = table;
|
||||
|
||||
}
|
||||
|
||||
chunkWords.add(current);
|
||||
|
||||
prev = current;
|
||||
}
|
||||
}
|
||||
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
chunkBlockList.add(chunkBlock);
|
||||
|
||||
document.setParagraphs(chunkBlockList);
|
||||
}
|
||||
|
||||
|
||||
private void mergeTableMetadata(Table currentTable, Table previousTable) {
|
||||
|
||||
// Distribute header information for subsequent tables
|
||||
if (previousTable != null && hasInvalidHeaderInformation(currentTable) && hasValidHeaderInformation(previousTable)) {
|
||||
List<Cell> previousTableNonHeaderRow = getRowWithNonHeaderCells(previousTable);
|
||||
List<Cell> tableNonHeaderRow = getRowWithNonHeaderCells(currentTable);
|
||||
// Allow merging of tables if header row is separated from first logical non-header row
|
||||
if (previousTableNonHeaderRow.isEmpty() && previousTable.getRowCount() == 1 && previousTable.getRows()
|
||||
.get(0)
|
||||
.size() == tableNonHeaderRow.size()) {
|
||||
previousTableNonHeaderRow = previousTable.getRows().get(0).stream().map(cell -> {
|
||||
Cell fakeCell = new Cell(cell.getPoints()[0], cell.getPoints()[2]);
|
||||
fakeCell.setHeaderCells(Collections.singletonList(cell));
|
||||
return fakeCell;
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
if (previousTableNonHeaderRow.size() == tableNonHeaderRow.size()) {
|
||||
for (int i = currentTable.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
|
||||
List<Cell> row = currentTable.getRows().get(i);
|
||||
if (row.size() == tableNonHeaderRow.size() && row.stream()
|
||||
.allMatch(cell -> cell.getHeaderCells().isEmpty())) {
|
||||
for (int j = 0; j < row.size(); j++) {
|
||||
row.get(j).setHeaderCells(previousTableNonHeaderRow.get(j).getHeaderCells());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Paragraph cb1 = buildTextBlock(chunkWords);
|
||||
if (cb1 != null) {
|
||||
chunkBlockList1.add(cb1);
|
||||
}
|
||||
|
||||
document.setParagraphs(chunkBlockList1);
|
||||
}
|
||||
|
||||
|
||||
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
|
||||
private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList) {
|
||||
|
||||
Paragraph paragraph = new Paragraph();
|
||||
TextBlock textBlock = null;
|
||||
@@ -110,54 +66,43 @@ public class SectionsBuilderService {
|
||||
boolean splitByTable = false;
|
||||
|
||||
Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
|
||||
boolean alreadyAdded = false;
|
||||
AbstractTextContainer previous = null;
|
||||
boolean alreadyAdded= false;
|
||||
while (itty.hasNext()) {
|
||||
AbstractTextContainer container = itty.next();
|
||||
|
||||
if (container instanceof Table) {
|
||||
Table table = (Table) container;
|
||||
splitByTable = true;
|
||||
|
||||
if (previous != null && previous.getText().startsWith("Table ")) {
|
||||
table.setHeadline(previous.getText());
|
||||
} else {
|
||||
table.setHeadline("Table in: " + lastHeadline);
|
||||
}
|
||||
|
||||
if (textBlock != null && !alreadyAdded) {
|
||||
paragraph.getPageBlocks().add(textBlock);
|
||||
alreadyAdded = true;
|
||||
alreadyAdded =true;
|
||||
}
|
||||
paragraph.getPageBlocks().add(table);
|
||||
paragraph.getPageBlocks().add(container);
|
||||
continue;
|
||||
}
|
||||
|
||||
TextBlock wordBlock = (TextBlock) container;
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
|
||||
.getSequences(), wordBlock.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
|
||||
textBlock.setPage(wordBlock.getPage());
|
||||
} else if (splitByTable) {
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
|
||||
.getSequences(), wordBlock.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
|
||||
textBlock.setPage(wordBlock.getPage());
|
||||
alreadyAdded = false;
|
||||
} else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
|
||||
textBlock.setPage(pageBefore);
|
||||
paragraph.getPageBlocks().add(textBlock);
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
|
||||
.getSequences(), wordBlock.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
|
||||
textBlock.setPage(wordBlock.getPage());
|
||||
} else {
|
||||
TextBlock spatialEntity = textBlock.union(wordBlock);
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
|
||||
.getHeight());
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
|
||||
spatialEntity.getWidth(), spatialEntity.getHeight());
|
||||
}
|
||||
pageBefore = wordBlock.getPage();
|
||||
splitByTable = false;
|
||||
previous = container;
|
||||
}
|
||||
|
||||
if (textBlock != null && !alreadyAdded) {
|
||||
@@ -167,41 +112,4 @@ public class SectionsBuilderService {
|
||||
}
|
||||
|
||||
|
||||
private boolean hasValidHeaderInformation(Table table) {
|
||||
|
||||
return !hasInvalidHeaderInformation(table);
|
||||
}
|
||||
|
||||
|
||||
private boolean hasInvalidHeaderInformation(Table table) {
|
||||
|
||||
return table.getRows().stream()
|
||||
.flatMap(row -> row.stream()
|
||||
.filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
.findAny()
|
||||
.isEmpty();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<Cell> getRowWithNonHeaderCells(Table table) {
|
||||
|
||||
for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
|
||||
List<Cell> row = table.getRows().get(i);
|
||||
boolean allNonHeader = true;
|
||||
for (Cell cell : row) {
|
||||
if (cell.isHeaderCell()) {
|
||||
allNonHeader = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (allNonHeader) {
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
return Collections.emptyList();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+6
-3
@@ -7,9 +7,12 @@ import lombok.Data;
|
||||
@Data
|
||||
@ConfigurationProperties("redaction-service")
|
||||
public class RedactionServiceSettings {
|
||||
|
||||
private int numberOfSurroundingWords = 3;
|
||||
|
||||
private int surroundingWordsOffsetWindow = 100;
|
||||
/**
|
||||
* Tenant used in single tenant mode.
|
||||
*/
|
||||
private String defaultTenant = "iqser-id";
|
||||
|
||||
private int flattenImageDpi = 100;
|
||||
|
||||
}
|
||||
-14
@@ -1,7 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
@@ -24,16 +22,4 @@ public abstract class AbstractTextContainer {
|
||||
return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
|
||||
}
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
}
|
||||
|
||||
public float getHeight() {
|
||||
return maxY - minY;
|
||||
}
|
||||
|
||||
public float getWidth() {
|
||||
return maxX - minX;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-51
@@ -2,12 +2,9 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
@@ -19,57 +16,11 @@ public class Cell extends Rectangle {
|
||||
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
|
||||
private List<Cell> headerCells = new ArrayList<>();
|
||||
|
||||
private boolean isHeaderCell;
|
||||
|
||||
|
||||
public Cell(Point2D topLeft, Point2D bottomRight) {
|
||||
|
||||
super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight
|
||||
.getY() - topLeft.getY()));
|
||||
super((float) topLeft.getY(), (float) topLeft.getX(), (float) (bottomRight.getX() - topLeft.getX()), (float) (bottomRight.getY() - topLeft.getY()));
|
||||
}
|
||||
|
||||
|
||||
public void addTextBlock(TextBlock textBlock) {
|
||||
|
||||
textBlocks.add(textBlock);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
TextPositionSequence previous = null;
|
||||
while (itty.hasNext()) {
|
||||
|
||||
TextBlock textBlock = itty.next();
|
||||
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.Value;
|
||||
|
||||
@Value
|
||||
@RequiredArgsConstructor
|
||||
public class CellPosition implements Comparable<CellPosition> {
|
||||
|
||||
int row;
|
||||
|
||||
int col;
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(CellPosition other) {
|
||||
|
||||
int rowDiff = row - other.row;
|
||||
return rowDiff != 0 ? rowDiff : col - other.col;
|
||||
}
|
||||
|
||||
}
|
||||
+10
-17
@@ -8,28 +8,25 @@ import org.locationtech.jts.index.strtree.STRtree;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
|
||||
@SuppressWarnings("all")
|
||||
public class RectangleSpatialIndex<T extends Rectangle> {
|
||||
|
||||
|
||||
private final STRtree si = new STRtree();
|
||||
private final List<T> rectangles = new ArrayList<>();
|
||||
|
||||
|
||||
public void add(T te) {
|
||||
|
||||
rectangles.add(te);
|
||||
si.insert(new Envelope(te.getLeft(), te.getRight(), te.getBottom(), te.getTop()), te);
|
||||
}
|
||||
|
||||
|
||||
public List<T> contains(Rectangle rectangle) {
|
||||
|
||||
List<T> intersection = si.query(new Envelope(rectangle.getLeft(), rectangle.getRight(), rectangle.getTop(), rectangle
|
||||
.getBottom()));
|
||||
|
||||
public List<T> contains(Rectangle r) {
|
||||
List<T> intersection = si.query(new Envelope(r.getLeft(), r.getRight(), r.getTop(), r.getBottom()));
|
||||
List<T> rv = new ArrayList<T>();
|
||||
|
||||
for (T ir : intersection) {
|
||||
if (rectangle.contains(ir)) {
|
||||
for (T ir: intersection) {
|
||||
if (r.contains(ir)) {
|
||||
rv.add(ir);
|
||||
}
|
||||
}
|
||||
@@ -37,22 +34,18 @@ public class RectangleSpatialIndex<T extends Rectangle> {
|
||||
Utils.sort(rv, Rectangle.ILL_DEFINED_ORDER);
|
||||
return rv;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<T> intersects(Rectangle r) {
|
||||
|
||||
List rv = si.query(new Envelope(r.getLeft(), r.getRight(), r.getTop(), r.getBottom()));
|
||||
return rv;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Minimum bounding box of all the Rectangles contained on this RectangleSpatialIndex
|
||||
*
|
||||
*
|
||||
* @return a Rectangle
|
||||
*/
|
||||
public Rectangle getBounds() {
|
||||
|
||||
return Rectangle.boundingBoxOf(rectangles);
|
||||
}
|
||||
|
||||
|
||||
+93
-154
@@ -9,38 +9,26 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@SuppressWarnings("all")
|
||||
public class Table extends AbstractTextContainer {
|
||||
|
||||
private final TreeMap<CellPosition, Cell> cells = new TreeMap<>();
|
||||
|
||||
private final RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
|
||||
private RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
private String headline;
|
||||
private int rowCount = 0;
|
||||
@Getter
|
||||
private int colCount = 0;
|
||||
|
||||
private int unrotatedRowCount;
|
||||
|
||||
private int unrotatedColCount;
|
||||
|
||||
private int rowCount = -1;
|
||||
|
||||
private int colCount = -1;
|
||||
|
||||
private final int rotation;
|
||||
|
||||
private List<List<Cell>> rows;
|
||||
private int rotation = 0;
|
||||
|
||||
private List<List<Cell>> memoizedRows = null;
|
||||
|
||||
public Table(List<Cell> cells, Rectangle area, int rotation) {
|
||||
|
||||
@@ -54,139 +42,43 @@ public class Table extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
|
||||
public List<List<Cell>> getRows() {
|
||||
|
||||
if (rows == null) {
|
||||
rows = computeRows();
|
||||
|
||||
// Ignore rows that does not contain any cells and values.
|
||||
List<List<Cell>> rowsToRemove = new ArrayList<>();
|
||||
for (List<Cell> row: rows){
|
||||
if (row.size() == 1 && row.get(0).getTextBlocks().isEmpty()){
|
||||
rowsToRemove.add(row);
|
||||
}
|
||||
}
|
||||
rows.removeAll(rowsToRemove);
|
||||
|
||||
computeHeaders();
|
||||
if (memoizedRows == null) {
|
||||
memoizedRows = computeRows();
|
||||
}
|
||||
|
||||
return rows;
|
||||
return memoizedRows;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public int getRowCount() {
|
||||
|
||||
if (rowCount == -1) {
|
||||
rowCount = getRows().size();
|
||||
}
|
||||
return rowCount;
|
||||
}
|
||||
|
||||
|
||||
public int getColCount() {
|
||||
|
||||
if (colCount == -1) {
|
||||
colCount = getRows().stream().mapToInt(List::size).max().orElse(0);
|
||||
}
|
||||
return colCount;
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Detect header cells (either first row or first column):
|
||||
* Column is marked as header if cell text is bold and row cell text is not bold.
|
||||
* Defaults to row.
|
||||
*/
|
||||
private void computeHeaders() {
|
||||
|
||||
if (rows == null) {
|
||||
rows = computeRows();
|
||||
}
|
||||
// A bold cell is a header cell as long as every cell to the left/top is bold, too
|
||||
// we move from left to right and top to bottom
|
||||
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
|
||||
List<Cell> rowCells = rows.get(rowIndex);
|
||||
for (int colIndex = 0; colIndex < rowCells.size(); colIndex++) {
|
||||
Cell cell = rowCells.get(colIndex);
|
||||
List<Cell> cellsToTheLeft = rowCells.subList(0, colIndex);
|
||||
Cell lastHeaderCell = null;
|
||||
for (Cell leftCell : cellsToTheLeft) {
|
||||
if (leftCell.isHeaderCell()) {
|
||||
lastHeaderCell = leftCell;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastHeaderCell != null) {
|
||||
cell.getHeaderCells().add(lastHeaderCell);
|
||||
}
|
||||
List<Cell> cellsToTheTop = new ArrayList<>();
|
||||
for (int i = 0; i < rowIndex; i++) {
|
||||
try {
|
||||
cellsToTheTop.add(rows.get(i).get(colIndex));
|
||||
} catch (IndexOutOfBoundsException e) {
|
||||
log.debug("No cell {} in row {}, ignoring.", colIndex, rowIndex);
|
||||
}
|
||||
}
|
||||
for (Cell topCell : cellsToTheTop) {
|
||||
if (topCell.isHeaderCell()) {
|
||||
lastHeaderCell = topCell;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastHeaderCell != null) {
|
||||
cell.getHeaderCells().add(lastHeaderCell);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(cell.getTextBlocks()) && cell.getTextBlocks()
|
||||
.get(0)
|
||||
.getMostPopularWordStyle()
|
||||
.equals("bold")) {
|
||||
cell.setHeaderCell(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<List<Cell>> computeRows() {
|
||||
|
||||
List<List<Cell>> rows = new ArrayList<>();
|
||||
if (rotation == 90) {
|
||||
for (int i = 0; i < unrotatedColCount; i++) { // rows
|
||||
for (int i = 0; i < colCount; i++) { // rows
|
||||
List<Cell> lastRow = new ArrayList<>();
|
||||
for (int j = unrotatedRowCount - 1; j >= 0; j--) { // cols
|
||||
for (int j = rowCount - 1; j >= 0; j--) { // cols
|
||||
Cell cell = cells.get(new CellPosition(j, i));
|
||||
if (cell != null) {
|
||||
lastRow.add(cell);
|
||||
}
|
||||
lastRow.add(cell);
|
||||
}
|
||||
rows.add(lastRow);
|
||||
}
|
||||
} else if (rotation == 270) {
|
||||
for (int i = unrotatedColCount - 1; i >= 0; i--) { // rows
|
||||
for (int i = colCount - 1; i >= 0; i--) { // rows
|
||||
List<Cell> lastRow = new ArrayList<>();
|
||||
for (int j = 0; j < unrotatedRowCount; j++) { // cols
|
||||
for (int j = 0; j < rowCount; j++) { // cols
|
||||
Cell cell = cells.get(new CellPosition(i, j));
|
||||
if (cell != null) {
|
||||
lastRow.add(cell);
|
||||
}
|
||||
lastRow.add(cell);
|
||||
}
|
||||
rows.add(lastRow);
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < unrotatedRowCount; i++) {
|
||||
for (int i = 0; i < rowCount; i++) {
|
||||
List<Cell> lastRow = new ArrayList<>();
|
||||
for (int j = 0; j < unrotatedColCount; j++) {
|
||||
for (int j = 0; j < colCount; j++) {
|
||||
Cell cell = cells.get(new CellPosition(i, j)); // JAVA_8 use getOrDefault()
|
||||
if (cell != null) {
|
||||
lastRow.add(cell);
|
||||
}
|
||||
lastRow.add(cell);
|
||||
}
|
||||
rows.add(lastRow);
|
||||
}
|
||||
@@ -196,18 +88,16 @@ public class Table extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
public void add(Cell chunk, int row, int col) {
|
||||
|
||||
private void add(Cell chunk, int row, int col) {
|
||||
|
||||
unrotatedRowCount = Math.max(unrotatedRowCount, row + 1);
|
||||
unrotatedColCount = Math.max(unrotatedColCount, col + 1);
|
||||
rowCount = Math.max(rowCount, row + 1);
|
||||
colCount = Math.max(colCount, col + 1);
|
||||
|
||||
CellPosition cp = new CellPosition(row, col);
|
||||
cells.put(cp, chunk);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void addCells(List<Cell> cells) {
|
||||
|
||||
if (cells.isEmpty()) {
|
||||
@@ -236,21 +126,25 @@ public class Table extends AbstractTextContainer {
|
||||
while (rowCells.hasNext()) {
|
||||
Cell cell = rowCells.next();
|
||||
if (i > 0) {
|
||||
Rectangle rectangle = new Rectangle(cell.getBottom(),
|
||||
si.getBounds().getLeft(),
|
||||
cell.getLeft() - si.getBounds().getLeft() + 1,
|
||||
si.getBounds().getBottom() - cell.getBottom());
|
||||
List<List<Cell>> others = rowsOfCells(si.contains(rectangle));
|
||||
List<List<Cell>> others = rowsOfCells(
|
||||
si.contains(
|
||||
new Rectangle(cell.getBottom(),
|
||||
si.getBounds().getLeft(),
|
||||
cell.getLeft() - si.getBounds().getLeft() + 1,
|
||||
si.getBounds().getBottom() - cell.getBottom()
|
||||
)
|
||||
));
|
||||
|
||||
for (List<Cell> r : others) {
|
||||
jumpToColumn = Math.max(jumpToColumn, r.size());
|
||||
}
|
||||
|
||||
while (startColumn != jumpToColumn) {
|
||||
add(previousNonNullCellForColumnIndex.get(startColumn), i, startColumn);
|
||||
startColumn++;
|
||||
}
|
||||
}
|
||||
|
||||
while (startColumn != jumpToColumn) {
|
||||
add(previousNonNullCellForColumnIndex.get(startColumn), i, startColumn);
|
||||
startColumn++;
|
||||
}
|
||||
|
||||
add(cell, i, startColumn);
|
||||
previousNonNullCellForColumnIndex.put(startColumn, cell);
|
||||
startColumn++;
|
||||
@@ -259,24 +153,34 @@ public class Table extends AbstractTextContainer {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<List<Cell>> rowsOfCells(List<Cell> cells) {
|
||||
|
||||
private static List<List<Cell>> rowsOfCells(List<Cell> cells) {
|
||||
Cell c;
|
||||
float lastTop;
|
||||
List<List<Cell>> rv = new ArrayList<>();
|
||||
List<Cell> lastRow;
|
||||
|
||||
if (cells.isEmpty()) {
|
||||
return rv;
|
||||
}
|
||||
cells.sort(Comparator.comparingDouble(Rectangle::getLeft));
|
||||
|
||||
cells.sort(Collections.reverseOrder((arg0, arg1) -> Float.compare(Utils.round(arg0.getBottom(), 2),
|
||||
Utils.round(arg1
|
||||
.getBottom(), 2))));
|
||||
Collections.sort(cells, new Comparator<Cell>() {
|
||||
@Override
|
||||
public int compare(Cell arg0, Cell arg1) {
|
||||
return Double.compare(arg0.getLeft(), arg1.getLeft());
|
||||
}
|
||||
});
|
||||
|
||||
Collections.sort(cells, Collections.reverseOrder(new Comparator<Cell>() {
|
||||
@Override
|
||||
public int compare(Cell arg0, Cell arg1) {
|
||||
return Float.compare(Utils.round(arg0.getBottom(), 2), Utils.round(arg1.getBottom(),2));
|
||||
}
|
||||
}));
|
||||
|
||||
Iterator<Cell> iter = cells.iterator();
|
||||
Cell c = iter.next();
|
||||
float lastTop = c.getBottom();
|
||||
List<Cell> lastRow = new ArrayList<>();
|
||||
c = iter.next();
|
||||
lastTop = c.getBottom();
|
||||
lastRow = new ArrayList<>();
|
||||
lastRow.add(c);
|
||||
rv.add(lastRow);
|
||||
|
||||
@@ -292,7 +196,6 @@ public class Table extends AbstractTextContainer {
|
||||
return rv;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getText() {
|
||||
|
||||
@@ -329,7 +232,6 @@ public class Table extends AbstractTextContainer {
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public String getTextAsHtml() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
@@ -363,4 +265,41 @@ public class Table extends AbstractTextContainer {
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
class CellPosition implements Comparable<CellPosition> {
|
||||
|
||||
CellPosition(int row, int col) {
|
||||
this.row = row;
|
||||
this.col = col;
|
||||
}
|
||||
|
||||
final int row, col;
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return row + 101 * col;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (obj == null) {
|
||||
return false;
|
||||
}
|
||||
if (getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
CellPosition other = (CellPosition) obj;
|
||||
return row == other.row && col == other.col;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(CellPosition other) {
|
||||
int rowdiff = row - other.row;
|
||||
return rowdiff != 0 ? rowdiff : col - other.col;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -18,9 +18,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
@Service
|
||||
public class RulingCleaningService {
|
||||
|
||||
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth){
|
||||
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float minCharHeight){
|
||||
if (!rulings.isEmpty()) {
|
||||
snapPoints(rulings, minCharWidth , minCharWidth);
|
||||
snapPoints(rulings, minCharWidth , minCharHeight);
|
||||
}
|
||||
|
||||
List<Ruling> vrs = new ArrayList<>();
|
||||
|
||||
+81
-74
@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.service;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -24,29 +25,26 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("all")
|
||||
public class TableExtractionService {
|
||||
|
||||
public void extractTables(CleanRulings cleanRulings, Page page) {
|
||||
public void extractTables(CleanRulings cleanRulings, Page page){
|
||||
|
||||
List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
|
||||
|
||||
List<TextBlock> toBeRemoved = new ArrayList<>();
|
||||
|
||||
for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
|
||||
TextBlock textBlock = (TextBlock) abstractTextContainer;
|
||||
Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
|
||||
while (itty.hasNext()) {
|
||||
TextBlock textBlock = (TextBlock) itty.next();
|
||||
for (Cell cell : cells) {
|
||||
if (cell.intersects(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight())) {
|
||||
cell.addTextBlock(textBlock);
|
||||
toBeRemoved.add(textBlock);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cells = new ArrayList<>(new HashSet<>(cells));
|
||||
Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
|
||||
|
||||
List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream()
|
||||
List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells)
|
||||
.stream()
|
||||
.filter(r -> r.getWidth() > 0f && r.getHeight() > 0f)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
@@ -65,32 +63,33 @@ public class TableExtractionService {
|
||||
for (Table table : tables) {
|
||||
int position = -1;
|
||||
|
||||
Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
|
||||
itty = page.getTextBlocks().iterator();
|
||||
while (itty.hasNext()) {
|
||||
AbstractTextContainer textBlock = itty.next();
|
||||
if (table.contains(textBlock) && position == -1) {
|
||||
position = page.getTextBlocks().indexOf(textBlock);
|
||||
AbstractTextContainer textBlock = (AbstractTextContainer) itty.next();
|
||||
if (table.contains(textBlock)) {
|
||||
if (position == -1) {
|
||||
position = page.getTextBlocks().indexOf(textBlock);
|
||||
}
|
||||
itty.remove();
|
||||
}
|
||||
}
|
||||
if (position != -1) {
|
||||
page.getTextBlocks().add(position, table);
|
||||
}
|
||||
}
|
||||
|
||||
page.getTextBlocks().removeAll(toBeRemoved);
|
||||
}
|
||||
|
||||
|
||||
public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
|
||||
|
||||
List<Cell> cellsFound = new ArrayList<>();
|
||||
Map<Point2D, Ruling[]> intersectionPoints = Ruling.findIntersections(horizontalRulingLines, verticalRulingLines);
|
||||
List<Point2D> intersectionPointsList = new ArrayList<>(intersectionPoints.keySet());
|
||||
intersectionPointsList.sort(POINT_COMPARATOR);
|
||||
Collections.sort(intersectionPointsList, POINT_COMPARATOR);
|
||||
boolean doBreak;
|
||||
|
||||
for (int i = 0; i < intersectionPointsList.size(); i++) {
|
||||
Point2D topLeft = intersectionPointsList.get(i);
|
||||
Ruling[] hv = intersectionPoints.get(topLeft);
|
||||
doBreak = false;
|
||||
|
||||
// CrossingPointsDirectlyBelow( topLeft );
|
||||
List<Point2D> xPoints = new ArrayList<>();
|
||||
@@ -107,6 +106,10 @@ public class TableExtractionService {
|
||||
}
|
||||
outer:
|
||||
for (Point2D xPoint : xPoints) {
|
||||
if (doBreak) {
|
||||
break;
|
||||
}
|
||||
|
||||
// is there a vertical edge b/w topLeft and xPoint?
|
||||
if (!hv[1].equals(intersectionPoints.get(xPoint)[1])) {
|
||||
continue;
|
||||
@@ -117,9 +120,11 @@ public class TableExtractionService {
|
||||
continue;
|
||||
}
|
||||
Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
|
||||
if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints
|
||||
.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
|
||||
if (intersectionPoints.containsKey(btmRight)
|
||||
&& intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0])
|
||||
&& intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
|
||||
cellsFound.add(new Cell(topLeft, btmRight));
|
||||
doBreak = true;
|
||||
break outer;
|
||||
}
|
||||
}
|
||||
@@ -134,7 +139,7 @@ public class TableExtractionService {
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
|
||||
public List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
|
||||
// via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
Set<Point2D> pointSet = new HashSet<>();
|
||||
@@ -142,6 +147,10 @@ public class TableExtractionService {
|
||||
Map<Point2D, Point2D> edgesV = new HashMap<>();
|
||||
int i = 0;
|
||||
|
||||
cells = new ArrayList<>(new HashSet<>(cells));
|
||||
|
||||
Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
|
||||
|
||||
for (Rectangle cell : cells) {
|
||||
for (Point2D pt : cell.getPoints()) {
|
||||
if (pointSet.contains(pt)) { // shared vertex, remove it
|
||||
@@ -154,10 +163,10 @@ public class TableExtractionService {
|
||||
|
||||
// X first sort
|
||||
List<Point2D> pointsSortX = new ArrayList<>(pointSet);
|
||||
pointsSortX.sort(X_FIRST_POINT_COMPARATOR);
|
||||
Collections.sort(pointsSortX, X_FIRST_POINT_COMPARATOR);
|
||||
// Y first sort
|
||||
List<Point2D> pointsSortY = new ArrayList<>(pointSet);
|
||||
pointsSortY.sort(POINT_COMPARATOR);
|
||||
Collections.sort(pointsSortY, POINT_COMPARATOR);
|
||||
|
||||
while (i < pointSet.size()) {
|
||||
float currY = (float) pointsSortY.get(i).getY();
|
||||
@@ -194,12 +203,13 @@ public class TableExtractionService {
|
||||
nextVertex = edgesV.get(curr.point);
|
||||
edgesV.remove(curr.point);
|
||||
lastAddedVertex = new PolygonVertex(nextVertex, Direction.VERTICAL);
|
||||
polygon.add(lastAddedVertex);
|
||||
} else {
|
||||
nextVertex = edgesH.get(curr.point);
|
||||
edgesH.remove(curr.point);
|
||||
lastAddedVertex = new PolygonVertex(nextVertex, Direction.HORIZONTAL);
|
||||
polygon.add(lastAddedVertex);
|
||||
}
|
||||
polygon.add(lastAddedVertex);
|
||||
|
||||
if (lastAddedVertex.equals(polygon.get(0))) {
|
||||
// closed polygon
|
||||
@@ -217,10 +227,10 @@ public class TableExtractionService {
|
||||
|
||||
// calculate grid-aligned minimum area rectangles for each found polygon
|
||||
for (List<PolygonVertex> poly : polygons) {
|
||||
float top = Float.MAX_VALUE;
|
||||
float left = Float.MAX_VALUE;
|
||||
float bottom = Float.MIN_VALUE;
|
||||
float right = Float.MIN_VALUE;
|
||||
float top = java.lang.Float.MAX_VALUE;
|
||||
float left = java.lang.Float.MAX_VALUE;
|
||||
float bottom = java.lang.Float.MIN_VALUE;
|
||||
float right = java.lang.Float.MIN_VALUE;
|
||||
for (PolygonVertex pt : poly) {
|
||||
top = (float) Math.min(top, pt.point.getY());
|
||||
left = (float) Math.min(left, pt.point.getX());
|
||||
@@ -234,66 +244,69 @@ public class TableExtractionService {
|
||||
}
|
||||
|
||||
|
||||
private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = (arg0, arg1) -> {
|
||||
private static final Comparator<Point2D> X_FIRST_POINT_COMPARATOR = new Comparator<Point2D>() {
|
||||
@Override
|
||||
public int compare(Point2D arg0, Point2D arg1) {
|
||||
int rv = 0;
|
||||
float arg0X = Utils.round(arg0.getX(), 2);
|
||||
float arg0Y = Utils.round(arg0.getY(), 2);
|
||||
float arg1X = Utils.round(arg1.getX(), 2);
|
||||
float arg1Y = Utils.round(arg1.getY(), 2);
|
||||
|
||||
int rv = 0;
|
||||
float arg0X = Utils.round(arg0.getX(), 2);
|
||||
float arg0Y = Utils.round(arg0.getY(), 2);
|
||||
float arg1X = Utils.round(arg1.getX(), 2);
|
||||
float arg1Y = Utils.round(arg1.getY(), 2);
|
||||
|
||||
if (arg0X > arg1X) {
|
||||
rv = 1;
|
||||
} else if (arg0X < arg1X) {
|
||||
rv = -1;
|
||||
} else if (arg0Y > arg1Y) {
|
||||
rv = 1;
|
||||
} else if (arg0Y < arg1Y) {
|
||||
rv = -1;
|
||||
if (arg0X > arg1X) {
|
||||
rv = 1;
|
||||
} else if (arg0X < arg1X) {
|
||||
rv = -1;
|
||||
} else if (arg0Y > arg1Y) {
|
||||
rv = 1;
|
||||
} else if (arg0Y < arg1Y) {
|
||||
rv = -1;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
return rv;
|
||||
};
|
||||
|
||||
private static final Comparator<Point2D> POINT_COMPARATOR = (arg0, arg1) -> {
|
||||
|
||||
int rv = 0;
|
||||
float arg0X = Utils.round(arg0.getX(), 2);
|
||||
float arg0Y = Utils.round(arg0.getY(), 2);
|
||||
float arg1X = Utils.round(arg1.getX(), 2);
|
||||
float arg1Y = Utils.round(arg1.getY(), 2);
|
||||
private static final Comparator<Point2D> POINT_COMPARATOR = new Comparator<Point2D>() {
|
||||
@Override
|
||||
public int compare(Point2D arg0, Point2D arg1) {
|
||||
int rv = 0;
|
||||
float arg0X = Utils.round(arg0.getX(), 2);
|
||||
float arg0Y = Utils.round(arg0.getY(), 2);
|
||||
float arg1X = Utils.round(arg1.getX(), 2);
|
||||
float arg1Y = Utils.round(arg1.getY(), 2);
|
||||
|
||||
if (arg0Y > arg1Y) {
|
||||
rv = 1;
|
||||
} else if (arg0Y < arg1Y) {
|
||||
rv = -1;
|
||||
} else if (arg0X > arg1X) {
|
||||
rv = 1;
|
||||
} else if (arg0X < arg1X) {
|
||||
rv = -1;
|
||||
|
||||
if (arg0Y > arg1Y) {
|
||||
rv = 1;
|
||||
} else if (arg0Y < arg1Y) {
|
||||
rv = -1;
|
||||
} else if (arg0X > arg1X) {
|
||||
rv = 1;
|
||||
} else if (arg0X < arg1X) {
|
||||
rv = -1;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
return rv;
|
||||
};
|
||||
|
||||
|
||||
private enum Direction {
|
||||
HORIZONTAL, VERTICAL
|
||||
HORIZONTAL,
|
||||
VERTICAL
|
||||
}
|
||||
|
||||
static class PolygonVertex {
|
||||
|
||||
Point2D point;
|
||||
Direction direction;
|
||||
|
||||
|
||||
PolygonVertex(Point2D point, Direction direction) {
|
||||
|
||||
public PolygonVertex(Point2D point, Direction direction) {
|
||||
this.direction = direction;
|
||||
this.point = point;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
|
||||
if (this == other) {
|
||||
return true;
|
||||
}
|
||||
@@ -303,21 +316,15 @@ public class TableExtractionService {
|
||||
return this.point.equals(((PolygonVertex) other).point);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
|
||||
return this.point.hashCode();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return String.format("%s[point=%s,direction=%s]", this.getClass()
|
||||
.getName(), this.point.toString(), this.direction.toString());
|
||||
return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-111
@@ -1,111 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Comparator;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Copied and minimal modified from PDFBox.
|
||||
*/
|
||||
public final class QuickSort {
|
||||
|
||||
private QuickSort() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static final Comparator<? extends Comparable> OBJCOMP = new Comparator<Comparable>() {
|
||||
@Override
|
||||
public int compare(Comparable object1, Comparable object2) {
|
||||
|
||||
return object1.compareTo(object2);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Sorts the given list using the given comparator.
|
||||
*
|
||||
* @param <T> type of the objects to be sorted.
|
||||
* @param list list to be sorted
|
||||
* @param cmp comparator used to compare the objects within the list
|
||||
*/
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> cmp) {
|
||||
|
||||
int size = list.size();
|
||||
if (size < 2) {
|
||||
return;
|
||||
}
|
||||
quicksort(list, cmp);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sorts the given list using compareTo as comparator.
|
||||
*
|
||||
* @param <T> type of the objects to be sorted.
|
||||
* @param list list to be sorted
|
||||
*/
|
||||
public static <T extends Comparable> void sort(List<T> list) {
|
||||
|
||||
sort(list, (Comparator<T>) OBJCOMP);
|
||||
}
|
||||
|
||||
|
||||
private static <T> void quicksort(List<T> list, Comparator<? super T> cmp) {
|
||||
|
||||
Deque<Integer> stack = new ArrayDeque<Integer>();
|
||||
stack.push(0);
|
||||
stack.push(list.size());
|
||||
while (!stack.isEmpty()) {
|
||||
int right = stack.pop();
|
||||
int left = stack.pop();
|
||||
if (right - left < 2) {
|
||||
continue;
|
||||
}
|
||||
int p = left + ((right - left) / 2);
|
||||
p = partition(list, cmp, p, left, right);
|
||||
|
||||
stack.push(p + 1);
|
||||
stack.push(right);
|
||||
|
||||
stack.push(left);
|
||||
stack.push(p);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static <T> int partition(List<T> list, Comparator<? super T> cmp, int p, int start, int end) {
|
||||
|
||||
int l = start;
|
||||
int h = end - 2;
|
||||
T piv = list.get(p);
|
||||
swap(list, p, end - 1);
|
||||
|
||||
while (l < h) {
|
||||
if (cmp.compare(list.get(l), piv) <= 0) {
|
||||
l++;
|
||||
} else if (cmp.compare(piv, list.get(h)) <= 0) {
|
||||
h--;
|
||||
} else {
|
||||
swap(list, l, h);
|
||||
}
|
||||
}
|
||||
int idx = h;
|
||||
if (cmp.compare(list.get(h), piv) < 0) {
|
||||
idx++;
|
||||
}
|
||||
swap(list, end - 1, idx);
|
||||
return idx;
|
||||
}
|
||||
|
||||
|
||||
private static <T> void swap(List<T> list, int i, int j) {
|
||||
|
||||
T tmp = list.get(i);
|
||||
list.set(i, list.get(j));
|
||||
list.set(j, tmp);
|
||||
}
|
||||
|
||||
}
|
||||
+6
-11
@@ -1,6 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.utils;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
@@ -12,29 +13,23 @@ public class Utils {
|
||||
|
||||
private final static float EPSILON = 0.1f;
|
||||
|
||||
|
||||
public static boolean feq(double f1, double f2) {
|
||||
|
||||
return (Math.abs(f1 - f2) < EPSILON);
|
||||
}
|
||||
|
||||
|
||||
public static float round(double d, int decimalPlace) {
|
||||
|
||||
BigDecimal bd = BigDecimal.valueOf(d);
|
||||
bd = bd.setScale(decimalPlace, BigDecimal.ROUND_HALF_UP);
|
||||
return bd.floatValue();
|
||||
}
|
||||
|
||||
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
|
||||
|
||||
public static <T> void sort(List<T> list, Comparator<? super T> comparator) {
|
||||
try {
|
||||
QuickSort.sort(list, comparator);
|
||||
} catch (IllegalArgumentException e) {
|
||||
// This should not happen since we use QuickSort from PDFBox
|
||||
Collections.sort(list, comparator);
|
||||
} catch (IllegalArgumentException e){
|
||||
//TODO Figure out why this happens.
|
||||
log.warn(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationHighlightService {
|
||||
|
||||
|
||||
public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
|
||||
if (!flatRedaction) {
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
|
||||
AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
|
||||
|
||||
if (textBlock.getPage() != page) {
|
||||
continue;
|
||||
}
|
||||
if (textBlock instanceof TextBlock) {
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream);
|
||||
} else if (textBlock instanceof Table) {
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
visualizeTable((Table) textBlock, contentStream);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
contentStream.close();
|
||||
}
|
||||
|
||||
if (classifiedDoc.getEntities().get(page) == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (Entity entity : classifiedDoc.getEntities().get(page)) {
|
||||
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
|
||||
if (flatRedaction && !isRedactionType(entity)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (TextPositionSequence textPositions : entityPositionSequence.getSequences()) {
|
||||
|
||||
float height = textPositions.getTextPositions().get(0).getHeightDir() + 2;
|
||||
|
||||
float posXInit;
|
||||
float posXEnd;
|
||||
float posYInit;
|
||||
float posYEnd;
|
||||
float[] quadPoints;
|
||||
|
||||
if (textPositions.getTextPositions().get(0).getRotation() == 90) {
|
||||
|
||||
posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
|
||||
posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
|
||||
posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
|
||||
posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 2;
|
||||
|
||||
quadPoints = new float[]{posXInit, posYInit, posXInit, posYEnd + height + 2, posXEnd, posYInit, posXEnd, posYEnd + height + 2};
|
||||
} else {
|
||||
|
||||
posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
|
||||
posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
|
||||
posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj();
|
||||
posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj();
|
||||
quadPoints = new float[]{posXInit, posYEnd + height + 2, posXEnd, posYEnd + height + 2, posXInit, posYInit - 2, posXEnd, posYEnd - 2};
|
||||
}
|
||||
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
|
||||
highlight.constructAppearances();
|
||||
|
||||
PDRectangle position = new PDRectangle();
|
||||
position.setLowerLeftX(posXInit);
|
||||
position.setLowerLeftY(posYEnd);
|
||||
position.setUpperRightX(posXEnd);
|
||||
position.setUpperRightY(posYEnd + height);
|
||||
|
||||
highlight.setRectangle(position);
|
||||
if (!flatRedaction) {
|
||||
highlight.setAnnotationName(entityPositionSequence.getId().toString());
|
||||
highlight.setTitlePopup(entityPositionSequence.getId().toString());
|
||||
highlight.setContents(entity.getRedactionReason());
|
||||
}
|
||||
|
||||
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
|
||||
// of the area to be highlighted
|
||||
|
||||
highlight.setQuadPoints(quadPoints);
|
||||
|
||||
PDColor color;
|
||||
if (flatRedaction) {
|
||||
color = new PDColor(new float[]{0, 0, 0}, PDDeviceRGB.INSTANCE);
|
||||
} else {
|
||||
color = new PDColor(getColor(entity), PDDeviceRGB.INSTANCE);
|
||||
}
|
||||
|
||||
highlight.setColor(color);
|
||||
annotations.add(highlight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private boolean isRedactionType(Entity entity) {
|
||||
if (!entity.isRedaction()) {
|
||||
return false;
|
||||
}
|
||||
if (entity.getType().equals(ADDRESS_CODE)) {
|
||||
return true;
|
||||
}
|
||||
if (entity.getType().equals(NAME_CODE)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(Entity entity) {
|
||||
if (!entity.isRedaction()) {
|
||||
return new float[]{0.627f, 0.627f, 0.627f};
|
||||
}
|
||||
if (entity.getType().equals(VERTEBRATES_CODE)) {
|
||||
return new float[]{0, 1, 0};
|
||||
}
|
||||
if (entity.getType().equals(ADDRESS_CODE)) {
|
||||
return new float[]{0, 1, 1};
|
||||
}
|
||||
if (entity.getType().equals(NAME_CODE)) {
|
||||
return new float[]{1, 1, 0};
|
||||
}
|
||||
if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
|
||||
return new float[]{1, 0.502f, 0};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.LIGHT_GRAY);
|
||||
contentStream.setLineWidth(0.5f);
|
||||
|
||||
contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
|
||||
contentStream.stroke();
|
||||
|
||||
if (textBlock.getClassification() != null) {
|
||||
contentStream.beginText();
|
||||
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
|
||||
contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
|
||||
|
||||
contentStream.showText(textBlock.getClassification());
|
||||
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
if (cell != null) {
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.setStrokingColor(Color.CYAN);
|
||||
contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell.getHeight());
|
||||
contentStream.stroke();
|
||||
|
||||
// contentStream.setStrokingColor(Color.GREEN);
|
||||
// for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
// contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
|
||||
// contentStream.stroke();
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (table.getClassification() != null) {
|
||||
contentStream.beginText();
|
||||
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
|
||||
contentStream.newLineAtOffset(table.getMinX(), table.getMinY());
|
||||
|
||||
contentStream.showText(table.getClassification());
|
||||
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
}
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.LosslessFactory;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.rendering.ImageType;
|
||||
import org.apache.pdfbox.rendering.PDFRenderer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PdfFlattenService {
|
||||
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
public PDDocument flattenPDF(PDDocument sourceDoc) throws IOException {
|
||||
|
||||
PDDocument destDoc = new PDDocument();
|
||||
|
||||
PDFRenderer pdfRenderer = new PDFRenderer(sourceDoc);
|
||||
|
||||
final int pageCount = sourceDoc.getDocumentCatalog().getPages().getCount();
|
||||
|
||||
log.info(pageCount + " page" + (pageCount == 1 ? "" : "s") + " to flatten.");
|
||||
|
||||
for (int i = 0; i < pageCount; i += 1) {
|
||||
|
||||
log.info("Flattening page " + (i + 1) + " of " + pageCount + "...");
|
||||
|
||||
BufferedImage img = pdfRenderer.renderImageWithDPI(i, settings.getFlattenImageDpi(), ImageType.RGB);
|
||||
|
||||
log.info("Image rendered in memory (" + img.getWidth() + "x" + img.getHeight() + " " + settings.getFlattenImageDpi() + "DPI). Adding to PDF...");
|
||||
|
||||
PDPage imagePage = new PDPage(new PDRectangle(img.getWidth(), img.getHeight()));
|
||||
destDoc.addPage(imagePage);
|
||||
|
||||
PDImageXObject imgObj = LosslessFactory.createFromImage(destDoc, img);
|
||||
|
||||
PDPageContentStream imagePageContentStream = new PDPageContentStream(destDoc, imagePage);
|
||||
imagePageContentStream.drawImage(imgObj, 0, 0);
|
||||
|
||||
log.info("Image added successfully.");
|
||||
|
||||
imagePageContentStream.close();
|
||||
|
||||
img.flush();
|
||||
}
|
||||
|
||||
log.info("New flattened PDF created in memory.");
|
||||
|
||||
sourceDoc.close();
|
||||
|
||||
return destDoc;
|
||||
}
|
||||
|
||||
}
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
server:
|
||||
port: 8083
|
||||
|
||||
configuration-service.url: "http://localhost:8081"
|
||||
@@ -1,8 +1,6 @@
|
||||
info:
|
||||
description: Redaction Service Server V1
|
||||
|
||||
configuration-service.url: "http://configuration-service-v1:8080"
|
||||
|
||||
server:
|
||||
port: 8080
|
||||
|
||||
@@ -10,10 +8,15 @@ spring:
|
||||
profiles:
|
||||
active: kubernetes
|
||||
|
||||
platform.multi-tenancy:
|
||||
enabled: ${multitenancy.enabled:false}
|
||||
tenantFilter:
|
||||
urlPatterns: /redact
|
||||
urlPatternsToIgnore:
|
||||
|
||||
management:
|
||||
endpoint:
|
||||
metrics.enabled: ${monitoring.enabled:false}
|
||||
prometheus.enabled: ${monitoring.enabled:false}
|
||||
health.enabled: true
|
||||
endpoints.web.exposure.include: prometheus, health
|
||||
metrics.export.prometheus.enabled: ${monitoring.enabled:false}
|
||||
@@ -2,6 +2,10 @@ spring:
|
||||
application:
|
||||
name: redaction-service-v1
|
||||
|
||||
management.endpoints:
|
||||
web.base-path: /
|
||||
enabled-by-default: false
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
base-path: /
|
||||
path-mapping:
|
||||
health: "health"
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package drools
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
|
||||
|
||||
global Section section
|
||||
|
||||
rule "0: Highlight Indicators"
|
||||
when
|
||||
eval(section.getEntities().isEmpty()==false);
|
||||
then
|
||||
section.highlightAll("VERTEBRATE");
|
||||
section.highlightAll("NO_REDACTION_INDICATOR");
|
||||
end
|
||||
|
||||
|
||||
rule "1: Redacted because Section contains Vertebrate"
|
||||
when
|
||||
eval(section.contains("VERTEBRATE")==true);
|
||||
then
|
||||
section.redact("NAME", 1, "Redacted because Section contains Vertebrate");
|
||||
section.redact("ADDRESS", 1, "Redacted because Section contains Vertebrate");
|
||||
end
|
||||
|
||||
|
||||
rule "2: Not Redacted because Section contains no Vertebrate"
|
||||
when
|
||||
eval(section.contains("VERTEBRATE")==false);
|
||||
then
|
||||
section.redactNot("NAME", 2, "Not Redacted because Section contains no Vertebrate");
|
||||
section.redactNot("ADDRESS", 2, "Not Redacted because Section contains no Vertebrate");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
eval(section.contains("VERTEBRATE")==true && section.contains("NO_REDACTION_INDICATOR")==true);
|
||||
then
|
||||
section.redactNot("NAME", 3, "Vertebrate was found, but also a no redaction indicator");
|
||||
section.redactNot("ADDRESS", 3, "Vertebrate was found, but also a no redaction indicator");
|
||||
end
|
||||
|
||||
|
||||
rule "4: Redact contact information, if applicant is found"
|
||||
when
|
||||
eval(section.getText().toLowerCase().contains("applicant"));
|
||||
then
|
||||
section.redactLineAfter("Name:", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
section.redactBetween("Address:", "Contact", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
section.redactLineAfter("Contact point:", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
section.redactLineAfter("Phone:", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
section.redactLineAfter("Fax:", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
section.redactLineAfter("E-mail:", "ADDRESS", 4, "Redacted because of Rule 4");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Redact contact information, if 'Producer of the plant protection product' is found"
|
||||
when
|
||||
eval(section.getText().contains("Producer of the plant protection product"));
|
||||
then
|
||||
section.redactLineAfter("Name:", "ADDRESS", 5, "xxxx");
|
||||
section.redactBetween("Address:", "Contact", "ADDRESS", 5, "xxxx");
|
||||
section.redactBetween("Contact:", "Phone", "ADDRESS", 5, "xxxx");
|
||||
section.redactLineAfter("Phone:", "ADDRESS", 5, "xxxx");
|
||||
section.redactLineAfter("Fax:", "ADDRESS", 5, "xxxx");
|
||||
section.redactLineAfter("E-mail:", "ADDRESS", 5, "xxxx");
|
||||
end
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class DummyTest {
|
||||
|
||||
@Test
|
||||
public void dummy(){
|
||||
System.out.println("Hello World");
|
||||
}
|
||||
}
|
||||
+17
-598
@@ -1,547 +1,47 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.DEFINED_PORT;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import com.iqser.red.service.redaction.v1.model.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.IdRemoval;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.Status;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
@Ignore
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(webEnvironment = RANDOM_PORT)
|
||||
@SpringBootTest(webEnvironment = DEFINED_PORT)
|
||||
public class RedactionIntegrationTest {
|
||||
|
||||
private static final String RULES = loadFromClassPath("drools/rules.drl");
|
||||
private static final String VERTEBRATE = "vertebrate";
|
||||
private static final String ADDRESS = "CBI_address";
|
||||
private static final String AUTHOR = "CBI_author";
|
||||
private static final String SPONSOR = "CBI_sponsor";
|
||||
private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
|
||||
private static final String REDACTION_INDICATOR = "redaction_indicator";
|
||||
private static final String HINT_ONLY = "hint_only";
|
||||
private static final String MUST_REDACT = "must_redact";
|
||||
private static final String PUBLISHED_INFORMATION = "published_information";
|
||||
private static final String TEST_METHOD = "test_method";
|
||||
private static final String PURITY = "purity";
|
||||
private static final String IMAGE = "image";
|
||||
|
||||
private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
|
||||
private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
|
||||
|
||||
private static final String FALSE_POSITIVE = "false_positive";
|
||||
|
||||
private static final String PII = "PII";
|
||||
|
||||
@Autowired
|
||||
private RedactionController redactionController;
|
||||
|
||||
@MockBean
|
||||
private RulesClient rulesClient;
|
||||
|
||||
@MockBean
|
||||
private DictionaryClient dictionaryClient;
|
||||
|
||||
private final Map<String, List<String>> dictionary = new HashMap<>();
|
||||
private final Map<String, String> typeColorMap = new HashMap<>();
|
||||
private final Map<String, Boolean> hintTypeMap = new HashMap<>();
|
||||
private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
|
||||
private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
|
||||
private final Map<String, Integer> rankTypeMap = new HashMap<>();
|
||||
private final Colors colors = new Colors();
|
||||
|
||||
private final static String TEST_RULESET_ID = "123";
|
||||
|
||||
@TestConfiguration
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Before
|
||||
public void stubClients() {
|
||||
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(RULES));
|
||||
|
||||
loadDictionaryForTest();
|
||||
loadTypeForTest();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(0L);
|
||||
when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(TypeResponse.builder()
|
||||
.types(getTypeResponse())
|
||||
.build());
|
||||
when(dictionaryClient.getDictionaryForType(VERTEBRATE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(VERTEBRATE));
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(SPONSOR));
|
||||
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
|
||||
when(dictionaryClient.getDictionaryForType(HINT_ONLY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(HINT_ONLY));
|
||||
when(dictionaryClient.getDictionaryForType(MUST_REDACT, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(MUST_REDACT));
|
||||
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
|
||||
when(dictionaryClient.getDictionaryForType(TEST_METHOD, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(TEST_METHOD));
|
||||
when(dictionaryClient.getDictionaryForType(PII, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PII));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
|
||||
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
|
||||
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
|
||||
when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
|
||||
when(dictionaryClient.getDictionaryForType(IMAGE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(IMAGE));
|
||||
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/hint_only.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/must_redact.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/published_information.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/test_method.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/PII.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_AUTHOR, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_author.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(RECOMMENDATION_ADDRESS, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/recommendation_CBI_address.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(FALSE_POSITIVE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/false_positive.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/purity.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
|
||||
.addAll(ResourceLoader.load("dictionaries/image.txt")
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
}
|
||||
|
||||
|
||||
private String cleanDictionaryEntry(String entry) {
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
|
||||
}
|
||||
|
||||
|
||||
private void loadTypeForTest() {
|
||||
|
||||
typeColorMap.put(VERTEBRATE, "#ff85f7");
|
||||
typeColorMap.put(ADDRESS, "#ffe187");
|
||||
typeColorMap.put(AUTHOR, "#ffe187");
|
||||
typeColorMap.put(SPONSOR, "#85ebff");
|
||||
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
|
||||
typeColorMap.put(REDACTION_INDICATOR, "#caff85");
|
||||
typeColorMap.put(HINT_ONLY, "#abc0c4");
|
||||
typeColorMap.put(MUST_REDACT, "#fab4c0");
|
||||
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
|
||||
typeColorMap.put(TEST_METHOD, "#91fae8");
|
||||
typeColorMap.put(PII, "#66ccff");
|
||||
typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
|
||||
typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
|
||||
typeColorMap.put(FALSE_POSITIVE, "#ffffff");
|
||||
typeColorMap.put(PURITY, "#ffe187");
|
||||
typeColorMap.put(IMAGE, "#fcc5fb");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
hintTypeMap.put(AUTHOR, false);
|
||||
hintTypeMap.put(SPONSOR, false);
|
||||
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(REDACTION_INDICATOR, true);
|
||||
hintTypeMap.put(HINT_ONLY, true);
|
||||
hintTypeMap.put(MUST_REDACT, true);
|
||||
hintTypeMap.put(PUBLISHED_INFORMATION, true);
|
||||
hintTypeMap.put(TEST_METHOD, true);
|
||||
hintTypeMap.put(PII, false);
|
||||
hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
hintTypeMap.put(FALSE_POSITIVE, true);
|
||||
hintTypeMap.put(PURITY, false);
|
||||
hintTypeMap.put(IMAGE, true);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
caseInSensitiveMap.put(AUTHOR, false);
|
||||
caseInSensitiveMap.put(SPONSOR, false);
|
||||
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
|
||||
caseInSensitiveMap.put(HINT_ONLY, true);
|
||||
caseInSensitiveMap.put(MUST_REDACT, true);
|
||||
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
|
||||
caseInSensitiveMap.put(TEST_METHOD, false);
|
||||
caseInSensitiveMap.put(PII, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
|
||||
caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
|
||||
caseInSensitiveMap.put(FALSE_POSITIVE, false);
|
||||
caseInSensitiveMap.put(PURITY, false);
|
||||
caseInSensitiveMap.put(IMAGE, true);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
recommendationTypeMap.put(AUTHOR, false);
|
||||
recommendationTypeMap.put(SPONSOR, false);
|
||||
recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(REDACTION_INDICATOR, false);
|
||||
recommendationTypeMap.put(HINT_ONLY, false);
|
||||
recommendationTypeMap.put(MUST_REDACT, false);
|
||||
recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
|
||||
recommendationTypeMap.put(TEST_METHOD, false);
|
||||
recommendationTypeMap.put(PII, false);
|
||||
recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
|
||||
recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
|
||||
recommendationTypeMap.put(FALSE_POSITIVE, false);
|
||||
recommendationTypeMap.put(PURITY, false);
|
||||
recommendationTypeMap.put(IMAGE, false);
|
||||
|
||||
|
||||
rankTypeMap.put(FALSE_POSITIVE, 160);
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
rankTypeMap.put(PII, 150);
|
||||
rankTypeMap.put(ADDRESS, 140);
|
||||
rankTypeMap.put(AUTHOR, 130);
|
||||
rankTypeMap.put(SPONSOR, 120);
|
||||
rankTypeMap.put(VERTEBRATE, 110);
|
||||
rankTypeMap.put(MUST_REDACT, 100);
|
||||
rankTypeMap.put(REDACTION_INDICATOR, 90);
|
||||
rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
|
||||
rankTypeMap.put(PUBLISHED_INFORMATION, 70);
|
||||
rankTypeMap.put(TEST_METHOD, 60);
|
||||
rankTypeMap.put(HINT_ONLY, 50);
|
||||
rankTypeMap.put(RECOMMENDATION_AUTHOR, 40);
|
||||
rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
|
||||
rankTypeMap.put(IMAGE, 30);
|
||||
|
||||
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
}
|
||||
|
||||
|
||||
private List<TypeResult> getTypeResponse() {
|
||||
|
||||
return typeColorMap.entrySet()
|
||||
.stream()
|
||||
.map(typeColor -> TypeResult.builder()
|
||||
.type(typeColor.getKey())
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
.build())
|
||||
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
private DictionaryResponse getDictionaryResponse(String type) {
|
||||
|
||||
return DictionaryResponse.builder()
|
||||
.hexColor(typeColorMap.get(type))
|
||||
.entries(dictionary.get(type))
|
||||
.isHint(hintTypeMap.get(type))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(type))
|
||||
.isRecommendation(recommendationTypeMap.get(type))
|
||||
.rank(rankTypeMap.get(type))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
System.out.println("noExceptionShouldBeThrownForAnyFiles");
|
||||
ClassLoader loader = getClass().getClassLoader();
|
||||
URL url = loader.getResource("files");
|
||||
File[] files = new File(url.getPath()).listFiles();
|
||||
List<File> input = new ArrayList<>();
|
||||
for (File file : files) {
|
||||
input.addAll(getPathsRecursively(file));
|
||||
}
|
||||
for (File path : input) {
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(new FileInputStream(path)))
|
||||
.build();
|
||||
System.out.println("Redacting file : " + path.getName());
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
|
||||
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
|
||||
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
||||
});
|
||||
|
||||
duplicates.entrySet().forEach(entry -> {
|
||||
assertThat(entry.getValue().size()).isEqualTo(1);
|
||||
});
|
||||
}
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<File> getPathsRecursively(File path) {
|
||||
|
||||
List<File> result = new ArrayList<>();
|
||||
if (path == null || path.listFiles() == null) {
|
||||
return result;
|
||||
}
|
||||
for (File f : path.listFiles()) {
|
||||
if (f.isFile()) {
|
||||
result.add(f);
|
||||
} else {
|
||||
result.addAll(getPathsRecursively(f));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
System.out.println("redactionTest");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
|
||||
request.setFlatRedaction(false);
|
||||
|
||||
RedactionResult result = redactionController.redact(request);
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTableRedaction() throws IOException {
|
||||
|
||||
System.out.println("testTableRedaction");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testManualRedaction() throws IOException {
|
||||
|
||||
System.out.println("testManualRedaction");
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
|
||||
|
||||
ManualRedactions manualRedactions = new ManualRedactions();
|
||||
|
||||
String manualAddId = UUID.randomUUID().toString();
|
||||
|
||||
Comment comment = Comment.builder()
|
||||
.date(OffsetDateTime.now())
|
||||
.user("TEST_USER")
|
||||
.text("This is a comment test")
|
||||
.build();
|
||||
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
|
||||
.id("0836727c3508a0b2ea271da69c04cc2f")
|
||||
.status(Status.REQUESTED)
|
||||
.build()));
|
||||
|
||||
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
|
||||
manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
|
||||
manualRedactions.getComments().put(manualAddId, List.of(comment));
|
||||
|
||||
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
|
||||
manualRedactionEntry.setId(manualAddId);
|
||||
manualRedactionEntry.setStatus(Status.REQUESTED);
|
||||
manualRedactionEntry.setType("name");
|
||||
manualRedactionEntry.setValue("O'Loughlin C.K.");
|
||||
manualRedactionEntry.setReason("Manual Redaction");
|
||||
manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1)));
|
||||
|
||||
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.manualRedactions(manualRedactions)
|
||||
.build();
|
||||
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
@@ -553,12 +53,9 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void classificationTest() throws IOException {
|
||||
|
||||
System.out.println("classificationTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
|
||||
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
|
||||
|
||||
RedactionResult result = redactionController.classify(request);
|
||||
|
||||
@@ -571,12 +68,9 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void sectionsTest() throws IOException {
|
||||
|
||||
System.out.println("sectionsTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " + "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
|
||||
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
|
||||
|
||||
RedactionResult result = redactionController.sections(request);
|
||||
|
||||
@@ -585,16 +79,12 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void htmlTablesTest() throws IOException {
|
||||
|
||||
System.out.println("htmlTablesTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/52 Fludioxonil_RAR_07_Volume_3CA_B-5_2018-02-21.pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
|
||||
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(request);
|
||||
|
||||
@@ -603,16 +93,12 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void htmlTableRotationTest() throws IOException {
|
||||
|
||||
System.out.println("htmlTableRotationTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
|
||||
RedactionRequest request = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(request);
|
||||
|
||||
@@ -621,71 +107,4 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void phantomCellsDocumentTest() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Phantom Cells.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
|
||||
if (!entry.isHint()) {
|
||||
assertThat(entry.getReason()).isEqualTo("Not redacted because row is not a vertebrate study");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void sponsorCompanyTest() throws IOException {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf");
|
||||
|
||||
AnalyzeRequest request = AnalyzeRequest.builder()
|
||||
.ruleSetId(TEST_RULESET_ID)
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
|
||||
AnalyzeResult result = redactionController.analyze(request);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
|
||||
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
|
||||
System.out.println("duration: " + (end - start));
|
||||
System.out.println("numberOfPages: " + result.getNumberOfPages());
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-520
@@ -1,520 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@SpringBootTest
|
||||
@RunWith(SpringRunner.class)
|
||||
public class EntityRedactionServiceTest {
|
||||
|
||||
private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
|
||||
private static final String AUTHOR_CODE = "author";
|
||||
private static final String ADDRESS_CODE = "address";
|
||||
private static final String SPONSOR_CODE = "sponsor";
|
||||
|
||||
private static final AtomicLong DICTIONARY_VERSION = new AtomicLong();
|
||||
private static final AtomicLong RULES_VERSION = new AtomicLong();
|
||||
|
||||
@MockBean
|
||||
private DictionaryClient dictionaryClient;
|
||||
|
||||
@MockBean
|
||||
private RulesClient rulesClient;
|
||||
|
||||
@Autowired
|
||||
private EntityRedactionService entityRedactionService;
|
||||
|
||||
@Autowired
|
||||
private PdfSegmentationService pdfSegmentationService;
|
||||
|
||||
@Autowired
|
||||
private DroolsExecutionService droolsExecutionService;
|
||||
|
||||
private final static String TEST_RULESET_ID = "123";
|
||||
|
||||
@TestConfiguration
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(DEFAULT_RULES.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testNestedEntitiesRemoval() {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false);
|
||||
Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false);
|
||||
entities.add(nested);
|
||||
entities.add(nesting);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
|
||||
assertThat(entities.size()).isEqualTo(1);
|
||||
assertThat(entities).contains(nesting);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTableRedaction() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testNestedRedaction() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/nested_redaction.pdf");
|
||||
|
||||
RedactionRequest redactionRequest = RedactionRequest.builder()
|
||||
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
|
||||
.build();
|
||||
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1)).hasSize(7);// 3 author cells, 1 address, 1 Y and 2 N entities
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTrueNegativesInTable() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/40 Cyprodinil - EU AIR3 - LCA Section 1" +
|
||||
" Supplement - Identity of the active substance - Reference list.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue();
|
||||
}
|
||||
pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " +
|
||||
"the plant protection product.pdf");
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFalsePositiveInWrongCell() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 9)
|
||||
.count()).isEqualTo(10);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testApplicantInTableRedaction() throws IOException {
|
||||
|
||||
String tableRules = "package drools\n" +
|
||||
"\n" +
|
||||
"import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
|
||||
"\n" +
|
||||
"global Section section\n" +
|
||||
"rule \"6: Redact contact information if applicant is found\"\n" +
|
||||
" when\n" +
|
||||
" eval(section.headlineContainsWord(\"applicant\") || section.getText().contains(\"Applicant\"));\n" +
|
||||
" then\n" +
|
||||
" section.redactLineAfter(\"Name:\", \"address\", 6,true, \"Applicant information was found\", \"Reg" +
|
||||
" (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactBetween(\"Address:\", \"Contact\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Contact point:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Phone:\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Fax:\", \"address\", 6,true, \"Applicant information was found\", \"Reg " +
|
||||
"(EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg" +
|
||||
" (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Tel:\", \"address\", 6,true, \"Applicant information was found\", \"Reg " +
|
||||
"(EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"E-mail:\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Email:\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Contact:\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Telephone number:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Fax number:\", \"address\", 6,true, \"Applicant information was found\"," +
|
||||
" \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactLineAfter(\"Telephone:\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactBetween(\"No:\", \"Fax\", \"address\", 6,true, \"Applicant information was found\", " +
|
||||
"\"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redactBetween(\"Contact:\", \"Tel.:\", \"address\", 6,true, \"Applicant information was found\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules(TEST_RULESET_ID);
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 6)
|
||||
.count()).isEqualTo(13);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSponsorInCell() throws IOException {
|
||||
|
||||
String tableRules = "package drools\n" +
|
||||
"\n" +
|
||||
"import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
|
||||
"\n" +
|
||||
"global Section section\n" + "rule \"11: Redact sponsor company\"\n" + " when\n" + " " +
|
||||
"Section(searchText.toLowerCase().contains(\"batches produced at\"))\n" + " then\n" + " section" +
|
||||
".redactIfPrecededBy(\"batches produced at\", \"sponsor\", 11, \"Redacted because it represents a " +
|
||||
"sponsor company\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" + " end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
droolsExecutionService.updateRules(TEST_RULESET_ID);
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse authorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(authorResponse);
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_sponsor.txt")))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream()
|
||||
.filter(entity -> entity.getMatchedRule() == 11)
|
||||
.count()).isEqualTo(1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void headerPropagation() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation.pdf");
|
||||
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Bissig R.", "Thanei P."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(2); // two pages
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(8);
|
||||
assertThat(classifiedDoc.getEntities().get(2).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(5); // 2 names, 1 address, 2 Y
|
||||
}
|
||||
|
||||
pdfFileResource = new ClassPathResource("files/Minimal Examples/Header Propagation2.pdf");
|
||||
|
||||
dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Arrays.asList("Tribolet, R.", "Muir, G.", "Kühne-Thu, H.", "Close, C."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 9).count()).isEqualTo(3);
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(9);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void testNGuideline() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Empty Tabular Data.pdf");
|
||||
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Aldershof S."))
|
||||
.build();
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.entries(Collections.emptyList())
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null);
|
||||
assertThat(classifiedDoc.getEntities()).hasSize(1); // one page
|
||||
assertThat(classifiedDoc.getEntities().get(1).stream().filter(entity -> entity.getMatchedRule() == 8).count()).isEqualTo(6);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Before
|
||||
public void stubRedaction() {
|
||||
String tableRules = "package drools\n" +
|
||||
"\n" +
|
||||
"import com.iqser.red.service.redaction.v1.server.redaction.model.Section\n" +
|
||||
"\n" +
|
||||
"global Section section\n" +
|
||||
"rule \"8: Not redacted because Vertebrate Study = N\"\n" +
|
||||
" when\n" +
|
||||
" Section(rowEquals(\"Vertebrate study Y/N\", \"N\") || rowEquals(\"Vertebrate study Y/N\", \"No\"))\n" +
|
||||
" then\n" +
|
||||
" section.redactNotCell(\"Author(s)\", 8, \"name\", false, \"Not redacted because row is not a vertebrate study\");\n" +
|
||||
" section.redactNot(\"address\", 8, \"Not redacted because row is not a vertebrate study\");\n" +
|
||||
" section.highlightCell(\"Vertebrate study Y/N\", 8, \"hint_only\");\n" +
|
||||
" end\n" +
|
||||
"rule \"9: Redact Authors and Addresses in Reference Table, if it is a Vertebrate study\"\n" +
|
||||
" when\n" +
|
||||
" Section(rowEquals(\"Vertebrate study Y/N\", \"Y\") || rowEquals(\"Vertebrate study Y/N\", " +
|
||||
"\"Yes\"))\n" +
|
||||
" then\n" +
|
||||
" section.redactCell(\"Author(s)\", 9, \"name\", false, \"Redacted because row is a vertebrate study\", \"Reg (EC) No 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.redact(\"address\", 9, \"Redacted because row is a vertebrate study\", \"Reg (EC) No" +
|
||||
" 1107/2009 Art. 63 (2g)\");\n" +
|
||||
" section.highlightCell(\"Vertebrate study Y/N\", 9, \"must_redact\");\n" +
|
||||
" end";
|
||||
when(rulesClient.getVersion(TEST_RULESET_ID)).thenReturn(RULES_VERSION.incrementAndGet());
|
||||
when(rulesClient.getRules(TEST_RULESET_ID)).thenReturn(new RulesResponse(tableRules));
|
||||
TypeResponse typeResponse = TypeResponse.builder()
|
||||
.types(Arrays.asList(
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(AUTHOR_CODE).hexColor("#ffff00").build(),
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(ADDRESS_CODE).hexColor("#ff00ff").build(),
|
||||
TypeResult.builder().ruleSetId(TEST_RULESET_ID).type(SPONSOR_CODE).hexColor("#00ffff").build()))
|
||||
.build();
|
||||
when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet());
|
||||
when(dictionaryClient.getAllTypes(TEST_RULESET_ID)).thenReturn(typeResponse);
|
||||
|
||||
// Default empty return to prevent NPEs
|
||||
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse);
|
||||
DictionaryResponse addressResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse);
|
||||
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
|
||||
.build();
|
||||
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse);
|
||||
|
||||
Colors colors = new Colors();
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
colors.setRequestAdd("#04b093");
|
||||
colors.setRequestRemove("#04b093");
|
||||
|
||||
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
public class RegExPatternTest {
|
||||
|
||||
|
||||
@Test
|
||||
public void testExpand(){
|
||||
String pattern = "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)";
|
||||
String text = ", G.R., Simoneaux,";
|
||||
Pattern compiledPattern = Pattern.compile(pattern, 0);
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(1);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testEmailRegEx(){
|
||||
String text = "Address: Schwarzwaldalle " +
|
||||
"P.O.Box\n" +
|
||||
"CH-4002 Basel\n" +
|
||||
"Switzerland\n" +
|
||||
"Contact: Christian Warmers\n" +
|
||||
"Tel: +41 (61) 323 8044\n" +
|
||||
"christian.warmers@syngenta.com";
|
||||
|
||||
|
||||
Pattern p = Pattern.compile("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
Matcher matcher = p.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(0);
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testEtAlRegEx() {
|
||||
String text = "To assess the potential of S-metolachlor to cause endocrine disruption (ED) a review (Charlton 2014,\n" +
|
||||
"ASB2016-762) was submitted that summarises results from regulatory and open scientific literature\n" +
|
||||
"studies covering in vitro and in vivo studies (level 2-5 of the OECD Conceptual Framework). According to this information metolachlor increased (1.5-fold) aromatase activity in JEG-3 cells (Laville et al.\n" +
|
||||
"2006, ASB2010-14391) and induced weak anti-androgenic activity in the MDA-kb2 reporter cell line\n" +
|
||||
"with a IC50 of 9.92 µM (IC50 of positive control flutamide: 0.51 µM) (Aït-Aïssa et al. 2010, ASB2015-\n" +
|
||||
"9562). Data from the Tox21 high throughput screening revealed just few postive findings in assays to\n" +
|
||||
"identify antagonists of the androgen receptor. An isolated result of this screening showed agonistic\n" +
|
||||
"activity on the thyroid stimulating hormone receptor, while Dalton et al. (2003, ASB2018-2832)\n" +
|
||||
"demonstrated that metolachlor induced CYP2B1/2 and CYP3A1/2 but did not affect T4, T3 or TSH.\n" +
|
||||
"After prepubertal exposure of male Wistar rats to metolachlor (Mathias et al. 2012, ASB2016-9890) a\n" +
|
||||
"statistically significant increase of serum hormone concentration was observed for testosterone (at the\n" +
|
||||
"dose 50 mg/kg) as well as a statistically significant decrease in the age of preputial separation at a dose\n" +
|
||||
"of 5 and 50 mg/kg. Furthermore a statistically significant increase for estradiol at a dose of 50 mg/kg\n" +
|
||||
"and for FSH at a dose of 5 and 50 mg/kg and morphological alterations of the seminiferous epithelium\n" +
|
||||
"were observed. Relative testicular weight was not altered. A statistically significant increase of relative\n" +
|
||||
"weights was observed in long-term studies with rats (Tisdel et al. 1983, TOX9800328 ). This finding\n" +
|
||||
"was attributed to lower terminal body weight. In mice a statistically significant decrease of the weight\n" +
|
||||
"seminal vesicle (Tisdel et al. 1982, TOX9800327) was shown after 24 month treatment with\n" +
|
||||
"metolachlor. In a mouse preimplantation embryo assay from open literature metolachlor increased the\n" +
|
||||
"percentage of apoptosis significantly and reduced the mean number of cells per embryo significantly\n" +
|
||||
"while the percentage of developing blastocytes was unaltered (Grennlee et al. 2004, ASB2016-9889).\n" +
|
||||
"In reproduvtive toxicity studies a retarded body weight development of the pups was observed, while\n" +
|
||||
"survival and normal morphological and functional development were not altered. No adverse effects\n" +
|
||||
"on male fertility were seen, however important parameters to assess effects on female fertility like\n" +
|
||||
"cyclicity, ovarian follicles as well as developmental landmarks in the offspring have not been investigated.";
|
||||
|
||||
Pattern p = Pattern.compile("([^\\s(]*?( \\w\\.?)?) et al\\.?");
|
||||
|
||||
Matcher matcher = p.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(1);
|
||||
System.out.println(match);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAuthorSplitting(){
|
||||
|
||||
String word = "Porch JR, " + "Kendall TZ, " + "Krueger HO";
|
||||
|
||||
word.replaceAll(",", " ").replaceAll(" ", " ");
|
||||
|
||||
Pattern pattern = Pattern.compile("[A-ZÄÖÜ][\\wäöüéèê]{2,}( [A-ZÄÖÜ]{1,2}\\.)+");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
|
||||
List<String> allMatches = new ArrayList<>();
|
||||
while (matcher.find()) {
|
||||
allMatches.add(matcher.group());
|
||||
}
|
||||
|
||||
for(String name: allMatches) {
|
||||
if(name.length() >= 3) {
|
||||
System.out.println(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
-64
@@ -1,64 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class ResourceLoader {
|
||||
|
||||
public Map<String, String> loadDictionaryFiles() {
|
||||
|
||||
String name = "dictionaries/";
|
||||
|
||||
List<String> files;
|
||||
try {
|
||||
files = IOUtils.readLines(ResourceLoader.class.getClassLoader().getResourceAsStream(name), "UTF-8");
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + name, e);
|
||||
}
|
||||
return files.stream().collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s));
|
||||
}
|
||||
|
||||
private String getFileName(String filePath) {
|
||||
return filePath.substring(0, filePath.indexOf(".txt"));
|
||||
}
|
||||
|
||||
public Set<String> load(String classpathPath) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
return br.lines().collect(Collectors.toSet());
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
|
||||
}
|
||||
}
|
||||
|
||||
public String loadToString(String classpathPath) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
return br.lines().collect(Collectors.joining("\n"));
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import org.assertj.core.api.Assertions;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TextNormalizationUtilitiesTest {
|
||||
|
||||
@Test
|
||||
public void testHyphenRemoval() {
|
||||
|
||||
String test = "Without these peo-\nple, this conference would not happen";
|
||||
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
|
||||
.contains("people");
|
||||
|
||||
test = "Die\t\nFreiwillige\t Versicherung\t endet\t zudem\t für\t den\t ein\u00AD\nzelnen\tVersicherten\tmit\tder\tAufhebung\tdes\tVertra-\nges,\t seiner\t Unterstellung\t unter\t die\t obligatorische\t\nVersicherung\t oder\t seinem\t Ausschluss.";
|
||||
Assertions.assertThat(TextNormalizationUtilities.removeHyphenLineBreaks(test))
|
||||
.contains("einzelnen", "Vertrages");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-189
@@ -1,189 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.mock.mockito.MockBean;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
|
||||
@SpringBootTest
|
||||
@RunWith(SpringRunner.class)
|
||||
public class PdfSegmentationServiceTest {
|
||||
|
||||
@Autowired
|
||||
private PdfSegmentationService pdfSegmentationService;
|
||||
|
||||
@Autowired
|
||||
private RulingCleaningService rulingCleaningService;
|
||||
|
||||
@Autowired
|
||||
private TableExtractionService tableExtractionService;
|
||||
|
||||
@Autowired
|
||||
private BlockificationService blockificationService;
|
||||
|
||||
@MockBean
|
||||
private KieContainer kieContainer;
|
||||
|
||||
|
||||
@Test
|
||||
public void testPDFSegmentationWithComplexTable() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document document = pdfSegmentationService.parseDocument(pdDocument);
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())).isNotEmpty();
|
||||
Table table = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(0);
|
||||
assertThat(table.getColCount()).isEqualTo(6);
|
||||
assertThat(table.getRowCount()).isEqualTo(13);
|
||||
assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTableExtraction() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document document = pdfSegmentationService.parseDocument(pdDocument);
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())).isNotEmpty();
|
||||
Table firstTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(0);
|
||||
assertThat(firstTable.getColCount()).isEqualTo(8);
|
||||
assertThat(firstTable.getRowCount()).isEqualTo(1);
|
||||
Table secondTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(1);
|
||||
assertThat(secondTable.getColCount()).isEqualTo(8);
|
||||
assertThat(secondTable.getRowCount()).isEqualTo(2);
|
||||
List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
|
||||
.get(0)
|
||||
.stream()
|
||||
.map(Collections::singletonList)
|
||||
.collect(Collectors.toList());
|
||||
assertThat(secondTable.getRows().stream()
|
||||
.allMatch(row -> row.stream()
|
||||
.map(Cell::getHeaderCells)
|
||||
.collect(Collectors.toList())
|
||||
.equals(firstTableHeaderCells)))
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMultiPageMetadataPropagation() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document document = pdfSegmentationService.parseDocument(pdDocument);
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())).isNotEmpty();
|
||||
Table firstTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(0);
|
||||
assertThat(firstTable.getColCount()).isEqualTo(9);
|
||||
assertThat(firstTable.getRowCount()).isEqualTo(5);
|
||||
Table secondTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(1);
|
||||
assertThat(secondTable.getColCount()).isEqualTo(9);
|
||||
assertThat(secondTable.getRowCount()).isEqualTo(6);
|
||||
List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
|
||||
.get(firstTable.getRowCount() - 1)
|
||||
.stream()
|
||||
.map(Cell::getHeaderCells)
|
||||
.collect(Collectors.toList());
|
||||
assertThat(secondTable.getRows().stream()
|
||||
.allMatch(row -> row.stream()
|
||||
.map(Cell::getHeaderCells)
|
||||
.collect(Collectors.toList())
|
||||
.equals(firstTableHeaderCells)))
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testHeaderCellsForRotatedTable() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
|
||||
Document document = pdfSegmentationService.parseDocument(pdDocument);
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())).isNotEmpty();
|
||||
Table firstTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(0);
|
||||
assertThat(firstTable.getColCount()).isEqualTo(8);
|
||||
assertThat(firstTable.getRowCount()).isEqualTo(1);
|
||||
Table secondTable = document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
.collect(Collectors.toList())
|
||||
.get(1);
|
||||
assertThat(secondTable.getColCount()).isEqualTo(8);
|
||||
assertThat(secondTable.getRowCount()).isEqualTo(6);
|
||||
List<List<Cell>> firstTableHeaderCells = firstTable.getRows()
|
||||
.get(0)
|
||||
.stream()
|
||||
.map(Collections::singletonList)
|
||||
.collect(Collectors.toList());
|
||||
assertThat(secondTable.getRows().stream()
|
||||
.allMatch(row -> row.stream()
|
||||
.map(Cell::getHeaderCells)
|
||||
.collect(Collectors.toList())
|
||||
.equals(firstTableHeaderCells)))
|
||||
.isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
configuration-service.url: "http://configuration-service-v1:8080"
|
||||
|
||||
ribbon:
|
||||
ConnectTimeout: 600000
|
||||
ReadTimeout: 600000
|
||||
|
||||
|
||||
spring:
|
||||
main:
|
||||
allow-bean-definition-overriding: true
|
||||
|
||||
-1654
File diff suppressed because it is too large
Load Diff
-8589
File diff suppressed because it is too large
Load Diff
-8
@@ -1,8 +0,0 @@
|
||||
Crumrine
|
||||
Fine Organics Limited, Middlesbrough, United Kingdom
|
||||
Hunan Haili Chemical Industry Co., Ltd., Hunan, China
|
||||
Monthey Syngenta Crop Protection AG, Basel, Switzerland
|
||||
Syngenta Crop Protection, Monthey, Switzerland
|
||||
Syngenta Monthey Switzerland
|
||||
Syngenta Nantong, China
|
||||
Syngenta, Switzerland
|
||||
-238
@@ -1,238 +0,0 @@
|
||||
(Parent
|
||||
ADME - Bioanalyses, Vergèze,
|
||||
Aeration: Gentle
|
||||
Andrews mean
|
||||
Andrews pKi
|
||||
BASF study code
|
||||
Billington –
|
||||
Bond elute
|
||||
Brown Liquid
|
||||
Brown Orange
|
||||
Brown Solid
|
||||
Brown disco
|
||||
Brown discoloured
|
||||
Brown fat
|
||||
Brown liquid
|
||||
Brown solid
|
||||
Buehler (1980),
|
||||
Buehler (1980).
|
||||
Buehler 9
|
||||
Buehler Test.
|
||||
Buehler assay
|
||||
Buehler test
|
||||
Buehler)
|
||||
Buehler-like :
|
||||
Calandra lark
|
||||
Cary, NC,
|
||||
Chambersas for
|
||||
Class 4
|
||||
Class III’
|
||||
Class III’)
|
||||
Class method,
|
||||
Class2 (effectively
|
||||
ClassMethod) Vischim
|
||||
Clay =
|
||||
Cochran-Armitage Test.
|
||||
Cochran-armitage test.
|
||||
Dunnett 2
|
||||
Dunnett's "t''
|
||||
Dunnett's Test
|
||||
Dunnett's multiple
|
||||
Dunnett's t-test
|
||||
Dunnett's test
|
||||
Dunnett) **
|
||||
Dunnett) Dose
|
||||
Dunnett) Food
|
||||
Dunnett) Lower
|
||||
Dunnett, 1964)
|
||||
Dunnett’s Multiple
|
||||
Dunnett’s T
|
||||
Dunnett’s Test)
|
||||
Dunnett’s adjustment
|
||||
Dunnett’s multiple
|
||||
Dunnett’s post
|
||||
Dunnett’s t-test)
|
||||
Fine vacuolation
|
||||
Fine white
|
||||
Finewhite powder
|
||||
Fischer 344
|
||||
Fischer 344)
|
||||
Fischer F344
|
||||
Fischer rat
|
||||
Fischer) Contaminants:
|
||||
Fischer-344 acceptability
|
||||
Fischer-344/
|
||||
Fischer-344/DuCrj acceptability
|
||||
Fischer344 rats.
|
||||
Fischer344) :
|
||||
Fischerrat embryo
|
||||
Fischer’s exact
|
||||
Fisher 344
|
||||
Fisher Exact Test
|
||||
Fisher Exact test
|
||||
Fisher exact test
|
||||
Fisher's Exact
|
||||
Fisher-344 rat
|
||||
Fisher344 (F344/
|
||||
Fisher344 rat
|
||||
Fisher344 rats
|
||||
Fisher344 rats.
|
||||
Fisher344 rats/
|
||||
Fisher344) group
|
||||
Fisher’s exact.
|
||||
Freeman-Tukey transform
|
||||
Gentle aeration
|
||||
Grade 2:
|
||||
Grade 3)
|
||||
Grade 3:
|
||||
Green
|
||||
Green Alga
|
||||
Green as
|
||||
Green-, blue-,
|
||||
GreenAlga, Selenastrum
|
||||
GreenAlga.. Springborn
|
||||
Gross External
|
||||
Gross Necropsy
|
||||
Gross and
|
||||
Gross necropsies
|
||||
Gross necropsy
|
||||
Gross pathology
|
||||
Gross pathology:
|
||||
Hall Farm
|
||||
Hall Farm)
|
||||
HallFarm (396
|
||||
Ham's F12
|
||||
Ham’s F10
|
||||
Hand-held
|
||||
HarlanWinkelmann, Borchem,
|
||||
Hill International,
|
||||
Hill Top
|
||||
Hill criteria
|
||||
Hill, Bracknell,
|
||||
HillTop Chamber,
|
||||
Hillmodel was
|
||||
Hill’s model
|
||||
Japanese White
|
||||
Jerneplaque assay
|
||||
Kent. Blood
|
||||
Klimisch score
|
||||
Litchfield and Wilcoxon
|
||||
Long term
|
||||
Long-Term Toxicity
|
||||
Long-term
|
||||
Longbw/
|
||||
Major Macroscopic findings
|
||||
Major biliary
|
||||
Major defects
|
||||
Major deviations
|
||||
Major fetal
|
||||
Major metabolic
|
||||
Major metabolites
|
||||
Major route
|
||||
Major treatment-related
|
||||
Mallard (Anas
|
||||
Mallard with
|
||||
Manual-Hand held
|
||||
Marsh Frog
|
||||
Marsh harrier
|
||||
Masonjars (500
|
||||
Meier formula)
|
||||
Meier survival
|
||||
Miles anti-Canine
|
||||
Miles anti-dog
|
||||
Netherlands, 2011
|
||||
Netherlands, published:
|
||||
Netherlands. Report
|
||||
Penn State
|
||||
Porewater:
|
||||
Rattray study
|
||||
Read across
|
||||
Reed Bunting
|
||||
Reichert-Jung Cryo
|
||||
RifconGmbH. Oxon
|
||||
Ritz and
|
||||
S.P.-U/
|
||||
San Diego,
|
||||
Shaw's Farm,
|
||||
Sprague
|
||||
Strain JapaneseWhite
|
||||
Strong morphological
|
||||
Weight -
|
||||
Weight 2744
|
||||
Weight Gain
|
||||
Weight On
|
||||
Weight Range
|
||||
Weight and
|
||||
Weight and length
|
||||
Weight at
|
||||
Weight gain
|
||||
Weight loss
|
||||
Weight of
|
||||
Weight only
|
||||
Weight range
|
||||
Weight, age
|
||||
Weight, feed
|
||||
Weight-of-evidence approach
|
||||
Weight:
|
||||
Weight; Adult
|
||||
Weight; Whole
|
||||
WeightGain 0-1
|
||||
WeightRange:
|
||||
Weightat dosing:
|
||||
Weightrange 200-250g.
|
||||
White (Tif:
|
||||
White 10
|
||||
White 3
|
||||
White Age
|
||||
White Powder
|
||||
White Strain
|
||||
White blood
|
||||
White cell
|
||||
White crystalline
|
||||
White guideline
|
||||
White guinea-pigs
|
||||
White hygroscopic
|
||||
White powder
|
||||
White solid
|
||||
White stork
|
||||
White swan
|
||||
White to off-white
|
||||
White wagtail
|
||||
White, solid
|
||||
WhitePowder Lot/
|
||||
Whiteguideline :
|
||||
Whitehygroscopic solid
|
||||
Whitepowder Lot/
|
||||
Whitepowder Stability
|
||||
Whitesolid Batch
|
||||
Whiteswan)
|
||||
Whiteto off-white
|
||||
Wildlife International specifications
|
||||
Wildlife Internationalspecifications Water:
|
||||
Williams E
|
||||
Williams medium
|
||||
Williams' medium
|
||||
WilliamsE medium
|
||||
Williams’ test
|
||||
Wilson's method.
|
||||
Wilson. All foetal
|
||||
Wilson.All foetal
|
||||
Wood pigeons
|
||||
York.
|
||||
Zeneca dossier);
|
||||
Zenecadossier); see
|
||||
green algae
|
||||
CTL
|
||||
No details reported
|
||||
Not given
|
||||
Not indicated
|
||||
Not reported
|
||||
Not reported.
|
||||
Not stated
|
||||
Syngenta
|
||||
Syngenta AG
|
||||
N/A
|
||||
No details reported
|
||||
Not available
|
||||
Test facility
|
||||
TBD
|
||||
-4
@@ -1,4 +0,0 @@
|
||||
Fundamentals of Applied Toxicology
|
||||
in vitro
|
||||
in-vitro
|
||||
published paper
|
||||
-87
@@ -1,87 +0,0 @@
|
||||
7th ed.; The Iowa State University Press: Ames, IA
|
||||
Analytical chemistry
|
||||
Animal Reproduction
|
||||
Animal Reproduction Science
|
||||
Apidologie
|
||||
Aquatic toxicology
|
||||
Archives of Environmental Contamination and Toxicology
|
||||
ATLA
|
||||
Atmospheric. Environment
|
||||
Australasian Journal of Ecotoxicology
|
||||
Background lesions of laboratory Animals, A Color Atlas
|
||||
Biometrics
|
||||
Biometrika
|
||||
Birth Defects Res. B. Dev. Reprod. Toxicol.
|
||||
Crit Rev Toxicol
|
||||
Current approaches in the statistical analysis of ecotoxicity data: guidance to application
|
||||
Curr. Med. Chem.
|
||||
Dongbei Nongye Daxue Xuebao
|
||||
Ecotoxicology and Environmental Safety
|
||||
Environ and Molecular Mutagenesis
|
||||
Environ. Health Perspec.
|
||||
Environ Health Perspect
|
||||
Environ Health Perspect.
|
||||
Environ Health Perspect. 1
|
||||
Environmental Health
|
||||
Environmental Health Perspectives
|
||||
Environmental & Molecular Mutagenesis
|
||||
Environmental Pollution
|
||||
Environmental Science
|
||||
Environmental Science and Technology.
|
||||
Environmental Science & Technology
|
||||
Environmental Toxicology and Chemistry
|
||||
Environ Mol Muta- gen
|
||||
Environ. Sci. Technol.
|
||||
Environ Toxicol.
|
||||
Env. Mol. Mutagen
|
||||
Erna¨hrung
|
||||
Essays in Honor of Harold Hotelling
|
||||
Fish Sci
|
||||
Food Cosmet. Toxicol.
|
||||
Fundamentals of Applied Toxicology
|
||||
Fundamentals of Applied Toxicology1988
|
||||
High-Throughput Screening Methods in Toxicity Testing
|
||||
High-Throughput Screening Methods in Toxicity Testing. Hoboken, NJ: John Wiley & Sons
|
||||
http://www.iobc-wprs.org
|
||||
Irish Journal of Agricultural and Food Research
|
||||
J Econom
|
||||
J Endocrinol
|
||||
J. Invest. Derm
|
||||
Journal of agricultural and food chemistry
|
||||
Journal of Applied Entomology
|
||||
Journal of Environmental Science and Health
|
||||
Journal of Experimental Biology and Ecology
|
||||
Journal of Hazardous Materials
|
||||
Journal of the American College of Toxicology
|
||||
Journal of Toxicology and
|
||||
J Steroid Biochem Mol Biol
|
||||
Marine Pollution Bulletin
|
||||
Middle Atlantic Reproduction and Teratology Association
|
||||
Mol. Cell. Endocrinol.
|
||||
Mol Mutagen
|
||||
Mutagenesis
|
||||
Mutat Res
|
||||
Nonparametric Statistics for the Behavioral Sciences
|
||||
Principles and Procedures of Statistics
|
||||
Principles and Procedures of Statistics, A Biometrical Approach
|
||||
Proc Natl Acad Sci USA
|
||||
Progress in
|
||||
Psychopharmacologia
|
||||
Reproductive Toxicology
|
||||
RNA
|
||||
Science of the Total Environment
|
||||
Stain Technol
|
||||
Statistical Methods
|
||||
Statistical Methods,
|
||||
Steinberg P,
|
||||
Teratology
|
||||
The American Statistician
|
||||
Toxicol Chem
|
||||
Toxicol in Vitro
|
||||
Toxicological and Environmental Chemistry
|
||||
Toxicological Sciences
|
||||
Toxicologic Pathology
|
||||
Toxicology and Applied Pharmacology
|
||||
Toxicol Sci
|
||||
Toxicol Sci.
|
||||
Toxicol Sci. 1
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
acute toxicity
|
||||
acute-toxicity
|
||||
dermal penetration
|
||||
eco toxicity
|
||||
eco-toxicity
|
||||
Env. Mol. Mutagen,
|
||||
in vivo
|
||||
in-vivo
|
||||
ld
|
||||
oral toxicity
|
||||
oral-toxicity
|
||||
Processes & Impacts
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user