Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 deiflaender
|
c9645c3329 |
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-parent</artifactId>
|
||||
<version>7.2.2</version>
|
||||
<version>7.1.2</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package buildjob;
|
||||
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
|
||||
import com.atlassian.bamboo.specs.api.BambooSpec;
|
||||
import com.atlassian.bamboo.specs.api.builders.BambooKey;
|
||||
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
|
||||
@@ -21,10 +23,6 @@ import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
|
||||
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
|
||||
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
|
||||
/**
|
||||
* Plan configuration for Bamboo.
|
||||
@@ -35,8 +33,6 @@ public class PlanSpec {
|
||||
|
||||
private static final String SERVICE_NAME = "redaction-service";
|
||||
|
||||
private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
|
||||
|
||||
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
|
||||
|
||||
/**
|
||||
@@ -76,6 +72,14 @@ public class PlanSpec {
|
||||
.jobs(new Job("Default Job",
|
||||
new BambooKey("JOB1"))
|
||||
.tasks(
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
"rm -rf ./*"),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout Default Repository")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
@@ -86,9 +90,20 @@ public class PlanSpec {
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME),
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml versions:set -DnewVersion=${bamboo.version_tag}; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml versions:set -DnewVersion=${bamboo.version_tag}; fi\n" +
|
||||
|
||||
"if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean install -Djava.security.egd=file:/dev/./urandom; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean deploy -e -DdeployAtEnd=true -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases; fi\n" +
|
||||
|
||||
"${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml package\n" +
|
||||
"${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml docker:push\n" +
|
||||
|
||||
"if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then echo \"gitTag=${bamboo.planRepository.1.branch}_${bamboo.buildNumber}\" > git.tag; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then echo \"gitTag=${bamboo.version_tag}\" > git.tag; fi\n"),
|
||||
createJUnitParserTask()
|
||||
.description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
SERVICE_NAME=$1
|
||||
|
||||
if [[ "${bamboo_version_tag}" = "dev" ]]
|
||||
then
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
clean install \
|
||||
-Djava.security.egd=file:/dev/./urandomelse
|
||||
else
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
--no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
versions:set \
|
||||
-DnewVersion=${bamboo_version_tag}
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
--no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
versions:set \
|
||||
-DnewVersion=${bamboo_version_tag}
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
clean deploy \
|
||||
-e \
|
||||
-DdeployAtEnd=true \
|
||||
-Dmaven.wagon.http.ssl.insecure=true \
|
||||
-Dmaven.wagon.http.ssl.allowall=true \
|
||||
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
|
||||
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
|
||||
fi
|
||||
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
--no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
package
|
||||
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
--no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
docker:push
|
||||
|
||||
if [[ "${bamboo_version_tag}" = "dev" ]]
|
||||
then
|
||||
echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
|
||||
else
|
||||
echo "gitTag=${bamboo_version_tag}" > git.tag
|
||||
fi
|
||||
@@ -5,7 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>platform-dependency</artifactId>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<version>1.1.2</version>
|
||||
<version>1.0.2</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<version>1.1.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
@@ -52,4 +52,4 @@
|
||||
|
||||
</dependencyManagement>
|
||||
|
||||
</project>
|
||||
</project>
|
||||
@@ -17,16 +17,5 @@
|
||||
<artifactId>spring-web</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
<version>2.11.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>file-management-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private String dossierTemplateId;
|
||||
private boolean reanalyseOnlyIfPossible;
|
||||
private ManualRedactions manualRedactions;
|
||||
private OffsetDateTime lastProcessed;
|
||||
|
||||
}
|
||||
|
||||
-30
@@ -1,30 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeResult {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private long duration;
|
||||
private int numberOfPages;
|
||||
private boolean hasHints;
|
||||
private boolean hasRequests;
|
||||
private boolean hasRedactions;
|
||||
private boolean hasImages;
|
||||
private boolean hasUpdates;
|
||||
private long dictionaryVersion;
|
||||
private long dossierDictionaryVersion;
|
||||
private long rulesVersion;
|
||||
private long legalBasisVersion;
|
||||
|
||||
|
||||
}
|
||||
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String dossierTemplateId;
|
||||
private String fileId;
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateResponse {
|
||||
|
||||
private byte[] document;
|
||||
}
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ChangeType {
|
||||
ADDED, REMOVED
|
||||
}
|
||||
+2
-2
@@ -1,12 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
|
||||
-19
@@ -1,19 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class ManualForceRedact {
|
||||
|
||||
private String id;
|
||||
private String user;
|
||||
private Status status;
|
||||
private String legalBasis;
|
||||
|
||||
}
|
||||
+3
-5
@@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@@ -27,6 +27,4 @@ public class ManualRedactionEntry {
|
||||
private String section;
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean addToDossierDictionary;
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ManualRedactionType {
|
||||
ADD, REMOVE, FORCE_REDACT
|
||||
ADD, REMOVE
|
||||
}
|
||||
|
||||
+5
-8
@@ -1,16 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@@ -20,9 +20,6 @@ public class ManualRedactions {
|
||||
@Builder.Default
|
||||
private Set<IdRemoval> idsToRemove = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<ManualForceRedact> forceRedacts = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
|
||||
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ReanalyzeResult {
|
||||
|
||||
private RedactionLog redactionLog;
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionChangeLog {
|
||||
|
||||
private List<RedactionChangeLogEntry> redactionLogEntry = new ArrayList<>();
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long dossierDictionaryVersion = -1;
|
||||
private long rulesVersion = -1;
|
||||
private long legalBasisVersion = -1;
|
||||
|
||||
}
|
||||
-47
@@ -1,47 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionChangeLogEntry {
|
||||
|
||||
private String id;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private int matchedRule;
|
||||
private String legalBasis;
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<Comment> comments = new ArrayList<>();
|
||||
|
||||
private ChangeType changeType;
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
}
|
||||
+15
-7
@@ -1,21 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.LegalBasisMapping;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLog {
|
||||
|
||||
public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion) {
|
||||
|
||||
this.redactionLogEntry = redactionLogEntry;
|
||||
this.dictionaryVersion = dictionaryVersion;
|
||||
this.rulesVersion = rulesVersion;
|
||||
}
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
private List<LegalBasisMapping> legalBasis;
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
private long dossierDictionaryVersion = -1;
|
||||
|
||||
private long rulesVersion = -1;
|
||||
private long legalBasisVersion = -1;
|
||||
|
||||
private String filename;
|
||||
|
||||
}
|
||||
|
||||
+3
-19
@@ -1,19 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class RedactionLogEntry {
|
||||
|
||||
private String id;
|
||||
@@ -34,19 +32,5 @@ public class RedactionLogEntry {
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
@Builder.Default
|
||||
private List<Comment> comments = new ArrayList<>();
|
||||
|
||||
private int startOffset;
|
||||
private int endOffset;
|
||||
|
||||
private boolean isImage;
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
}
|
||||
|
||||
+2
-3
@@ -11,8 +11,7 @@ import lombok.NoArgsConstructor;
|
||||
@AllArgsConstructor
|
||||
public class RedactionRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private String dossierTemplateId;
|
||||
private byte[] document;
|
||||
private boolean flatRedaction;
|
||||
private ManualRedactions manualRedactions;
|
||||
}
|
||||
|
||||
+2
@@ -13,5 +13,7 @@ public class RedactionResult {
|
||||
|
||||
private byte[] document;
|
||||
private int numberOfPages;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
|
||||
}
|
||||
|
||||
-33
@@ -1,33 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SectionArea {
|
||||
|
||||
@NonNull
|
||||
private Point topLeft;
|
||||
|
||||
@NonNull
|
||||
private float width;
|
||||
|
||||
@NonNull
|
||||
private float height;
|
||||
|
||||
@NonNull
|
||||
private int page;
|
||||
|
||||
private String header;
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
return page == other.getPage() && this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight();
|
||||
}
|
||||
|
||||
}
|
||||
+4
-4
@@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
|
||||
+8
-14
@@ -1,21 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
String SERVICE_NAME = "redaction-service-v1";
|
||||
|
||||
String RULE_SET_PARAMETER_NAME = "dossierTemplateId";
|
||||
String RULE_SET_PATH_VARIABLE = "/{" + RULE_SET_PARAMETER_NAME + "}";
|
||||
|
||||
|
||||
@PostMapping(value = "/annotate", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest);
|
||||
@PostMapping(value = "/redact", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult redact(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
|
||||
@@ -26,10 +23,7 @@ public interface RedactionResource {
|
||||
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/rules/update" + RULE_SET_PATH_VARIABLE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId);
|
||||
@PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@RequestBody String rules);
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void testRules(@RequestBody String rules);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,6 @@
|
||||
<artifactId>redaction-service-server-v1</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<artifactId>storage-commons</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
@@ -23,18 +19,8 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>file-management-service-api-v1</artifactId>
|
||||
<version>2.25.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
<version>1.3.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.drools</groupId>
|
||||
@@ -56,23 +42,6 @@
|
||||
<artifactId>guava</artifactId>
|
||||
<version>29.0-jre</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>jbig2-imageio</artifactId>
|
||||
<version>3.0.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.jai-imageio</groupId>
|
||||
<artifactId>jai-imageio-core</artifactId>
|
||||
<version>1.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.jai-imageio</groupId>
|
||||
<artifactId>jai-imageio-jpeg2000</artifactId>
|
||||
<version>1.4.0</version>
|
||||
</dependency>
|
||||
|
||||
<!-- commons -->
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
@@ -104,13 +73,11 @@
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-openfeign</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- ribbon must be included because transitive a dependency on it.-->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-amqp</artifactId>
|
||||
<version>2.3.1.RELEASE</version>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-netflix-ribbon</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- test dependencies -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -122,12 +89,6 @@
|
||||
<artifactId>test-commons</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.amqp</groupId>
|
||||
<artifactId>spring-rabbit-test</artifactId>
|
||||
<version>2.3.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
|
||||
+51
-6
@@ -1,26 +1,71 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
|
||||
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
@Import({DefaultWebMvcConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
|
||||
public class Application {
|
||||
|
||||
@Autowired
|
||||
private RulesClient rulesClient;
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.setProperty("org.apache.pdfbox.rendering.UsePureJavaCMYKConversion", "true");
|
||||
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
try {
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
RulesResponse rules = rulesClient.getRules();
|
||||
if (StringUtils.isEmpty(rules.getRules())) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+10
-20
@@ -1,18 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@@ -20,23 +18,15 @@ public class Document {
|
||||
|
||||
private List<Page> pages = new ArrayList<>();
|
||||
private List<Paragraph> paragraphs = new ArrayList<>();
|
||||
private List<Header> headers = new ArrayList<>();
|
||||
private List<Footer> footers = new ArrayList<>();
|
||||
private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
|
||||
private Map<Integer, List<Entity>> entities = new HashMap<>();
|
||||
private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter = new FloatFrequencyCounter();
|
||||
private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
|
||||
private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
|
||||
private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
|
||||
private boolean headlines;
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
private SectionGrid sectionGrid = new SectionGrid();
|
||||
private DictionaryVersion dictionaryVersion;
|
||||
private long dictionaryVersion;
|
||||
private long rulesVersion;
|
||||
|
||||
private List<SectionText> sectionText = new ArrayList<>();
|
||||
|
||||
private Map<Integer, Set<Image>> images = new HashMap<>();
|
||||
|
||||
}
|
||||
|
||||
+22
-20
@@ -1,7 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
@@ -9,35 +7,38 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class FloatFrequencyCounter {
|
||||
import lombok.Getter;
|
||||
|
||||
public class FloatFrequencyCounter
|
||||
{
|
||||
|
||||
@Getter
|
||||
Map<Float, Integer> countPerValue = new HashMap<>();
|
||||
|
||||
public void add(float value) {
|
||||
if (!countPerValue.containsKey(value)) {
|
||||
public void add(float value){
|
||||
if(!countPerValue.containsKey(value)){
|
||||
countPerValue.put(value, 1);
|
||||
} else {
|
||||
countPerValue.put(value, countPerValue.get(value) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
public void addAll(Map<Float, Integer> otherCounter) {
|
||||
for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
|
||||
if (countPerValue.containsKey(entry.getKey())) {
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
|
||||
public void addAll(Map<Float, Integer> otherCounter){
|
||||
for(Map.Entry<Float, Integer> entry: otherCounter.entrySet()){
|
||||
if(countPerValue.containsKey(entry.getKey())){
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey())+ entry.getValue());
|
||||
} else {
|
||||
countPerValue.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Float getMostPopular() {
|
||||
public Float getMostPopular(){
|
||||
Map.Entry<Float, Integer> mostPopular = null;
|
||||
for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
|
||||
if (mostPopular == null) {
|
||||
for(Map.Entry<Float, Integer> entry: countPerValue.entrySet()){
|
||||
if(mostPopular == null){
|
||||
mostPopular = entry;
|
||||
} else if (entry.getValue() >= mostPopular.getValue()) {
|
||||
} else if(entry.getValue() >= mostPopular.getValue()){
|
||||
mostPopular = entry;
|
||||
}
|
||||
}
|
||||
@@ -45,11 +46,12 @@ public class FloatFrequencyCounter {
|
||||
}
|
||||
|
||||
|
||||
public List<Float> getHighterThanMostPopular() {
|
||||
|
||||
public List<Float> getHighterThanMostPopular(){
|
||||
Float mostPopular = getMostPopular();
|
||||
List<Float> higher = new ArrayList<>();
|
||||
for (Float value : countPerValue.keySet()) {
|
||||
if (value > mostPopular) {
|
||||
for(Float value: countPerValue.keySet()){
|
||||
if(value > mostPopular){
|
||||
higher.add(value);
|
||||
}
|
||||
}
|
||||
@@ -58,12 +60,12 @@ public class FloatFrequencyCounter {
|
||||
}
|
||||
|
||||
|
||||
public Float getHighest() {
|
||||
public Float getHighest(){
|
||||
Float highest = null;
|
||||
for (Float value : countPerValue.keySet()) {
|
||||
if (highest == null) {
|
||||
for(Float value: countPerValue.keySet()){
|
||||
if (highest == null){
|
||||
highest = value;
|
||||
} else if (value > highest) {
|
||||
} else if(value > highest){
|
||||
highest = value;
|
||||
}
|
||||
}
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Footer {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
@JsonIgnore
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Header {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
@JsonIgnore
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-6
@@ -1,6 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
public enum Orientation {
|
||||
|
||||
NONE, LEFT, RIGHT
|
||||
}
|
||||
+4
-10
@@ -1,14 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class Page {
|
||||
@@ -16,8 +16,6 @@ public class Page {
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
|
||||
private List<PdfImage> images;
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
private boolean landscape;
|
||||
@@ -30,12 +28,8 @@ public class Page {
|
||||
private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
|
||||
private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
|
||||
|
||||
private double cropBoxArea;
|
||||
|
||||
|
||||
public boolean isRotated() {
|
||||
|
||||
return rotation != 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+9
-17
@@ -1,21 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class Paragraph implements Comparable {
|
||||
public class Paragraph {
|
||||
|
||||
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private String headline;
|
||||
|
||||
|
||||
@@ -54,11 +53,4 @@ public class Paragraph implements Comparable {
|
||||
return textBlocks;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(Object o) {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SectionText {
|
||||
|
||||
private int sectionNumber;
|
||||
private String text;
|
||||
|
||||
private boolean isTable;
|
||||
private String headline;
|
||||
|
||||
private List<SectionArea> sectionAreas = new ArrayList<>();
|
||||
private Set<Image> images = new HashSet<>();
|
||||
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
private Map<String, CellValue> tabularData = new HashMap<>();
|
||||
private List<Integer> cellStarts = new ArrayList<>();
|
||||
|
||||
|
||||
public void setTabularData(Map<String, CellValue> tabularData) {
|
||||
tabularData.remove(null);
|
||||
this.tabularData = tabularData;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> {
|
||||
if (block != null) {
|
||||
searchableText.addAll(block.getSequences());
|
||||
}
|
||||
});
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
+3
-3
@@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class StringFrequencyCounter {
|
||||
|
||||
@Getter
|
||||
@@ -46,4 +46,4 @@ public class StringFrequencyCounter {
|
||||
return mostPopular != null ? mostPopular.getKey() : null;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Text {
|
||||
|
||||
private int numberOfPages;
|
||||
private List<SectionText> sectionTexts = new ArrayList<>();
|
||||
|
||||
}
|
||||
+10
-13
@@ -1,21 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
@Builder.Default
|
||||
@@ -32,7 +30,6 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
private String classification;
|
||||
|
||||
|
||||
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
|
||||
this.minX = minX;
|
||||
this.maxX = maxX;
|
||||
@@ -101,6 +98,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
@@ -120,7 +118,6 @@ public class TextBlock extends AbstractTextContainer {
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonIgnore
|
||||
public String getText() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
@@ -142,4 +139,4 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class UnclassifiedText {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
@JsonIgnore
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
+14
-99
@@ -1,11 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
@@ -13,20 +17,11 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("all")
|
||||
public class BlockificationService {
|
||||
|
||||
static final float THRESHOLD = 1f;
|
||||
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
|
||||
List<Ruling> verticalRulingLines) {
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
|
||||
|
||||
List<TextPositionSequence> chunkWords = new ArrayList<>();
|
||||
List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
|
||||
@@ -34,46 +29,21 @@ public class BlockificationService {
|
||||
float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
|
||||
TextPositionSequence prev = null;
|
||||
|
||||
boolean wasSplitted = false;
|
||||
Float splitX1 = null;
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
|
||||
boolean startFromTop = word.getY1() > maxY + word.getHeight();
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
|
||||
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
|
||||
boolean splittedByRuling = word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines) || word
|
||||
.getRotation() == 0 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines) || word
|
||||
.getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
|
||||
.getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
|
||||
|
||||
Orientation prevOrientation = null;
|
||||
if(!chunkBlockList1.isEmpty()) {
|
||||
prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
|
||||
}
|
||||
if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), verticalRulingLines))) {
|
||||
|
||||
TextBlock cb1 = buildTextBlock(chunkWords);
|
||||
chunkBlockList1.add(cb1);
|
||||
chunkWords = new ArrayList<>();
|
||||
|
||||
if (splitByX && !splittedByRuling) {
|
||||
wasSplitted = true;
|
||||
cb1.setOrientation(Orientation.LEFT);
|
||||
splitX1 = word.getX1();
|
||||
} else
|
||||
|
||||
if (newLineAfterSplit && !splittedByRuling) {
|
||||
wasSplitted = false;
|
||||
cb1.setOrientation(Orientation.RIGHT);
|
||||
splitX1 = null;
|
||||
} else
|
||||
|
||||
if(prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !splittedByRuling)){
|
||||
cb1.setOrientation(Orientation.LEFT);
|
||||
}
|
||||
|
||||
minX = 1000;
|
||||
maxX = 0;
|
||||
minY = 1000;
|
||||
@@ -103,62 +73,9 @@ public class BlockificationService {
|
||||
chunkBlockList1.add(cb1);
|
||||
}
|
||||
|
||||
Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
|
||||
|
||||
TextBlock previousLeft = null;
|
||||
TextBlock previousRight = null;
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if(previousLeft != null && block.getOrientation().equals(Orientation.LEFT)){
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()){
|
||||
previousLeft.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(previousRight != null && block.getOrientation().equals(Orientation.RIGHT)){
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()){
|
||||
previousRight.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (block.getOrientation().equals(Orientation.LEFT)) {
|
||||
previousLeft = block;
|
||||
} else if (block.getOrientation().equals(Orientation.RIGHT)) {
|
||||
previousRight = block;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
itty = chunkBlockList1.iterator();
|
||||
TextBlock previous = null;
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if(previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous
|
||||
.getMaxY())||
|
||||
previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous
|
||||
.getMaxY())){
|
||||
previous.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
|
||||
previous = block;
|
||||
}
|
||||
|
||||
|
||||
return new Page(chunkBlockList1);
|
||||
}
|
||||
|
||||
private boolean equalsWithThreshold(float f1, float f2){
|
||||
return Math.abs(f1 - f2) < THRESHOLD;
|
||||
}
|
||||
|
||||
|
||||
private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {
|
||||
|
||||
@@ -201,8 +118,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
|
||||
List<Ruling> rulingLines) {
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
|
||||
|
||||
for (Ruling ruling : rulingLines) {
|
||||
if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
|
||||
@@ -213,8 +129,7 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
|
||||
boolean landscape) {
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
|
||||
+28
-54
@@ -1,19 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ClassificationService {
|
||||
@@ -28,7 +28,7 @@ public class ClassificationService {
|
||||
|
||||
List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
|
||||
|
||||
log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
|
||||
System.out.println(document.getFontSizeCounter().getCountPerValue());
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
@@ -39,7 +39,6 @@ public class ClassificationService {
|
||||
|
||||
|
||||
public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
|
||||
@@ -48,33 +47,24 @@ public class ClassificationService {
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// TODO Figure out why this happens.
|
||||
return;
|
||||
}
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Header");
|
||||
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1
|
||||
&& (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
|
||||
&& PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@@ -82,36 +72,20 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
} else {
|
||||
textBlock.setClassification("Other");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
|
||||
+4
-2
@@ -1,8 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface DictionaryClient extends DictionaryResource {
|
||||
}
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
|
||||
import com.iqser.red.service.file.management.v1.api.resources.FileStatusProcessingUpdateResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${file-management-service.url}")
|
||||
public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@FeignClient(name = "ImageClassificationResource", url = "${image-service.url}")
|
||||
public interface ImageClassificationClient {
|
||||
|
||||
@PostMapping(value = "/process_full_img", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
ImageClassificationResponse classify(@RequestBody MultipartFile file);
|
||||
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImageClassificationResponse {
|
||||
|
||||
private String category;
|
||||
}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.LegalBasisMappingResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "LegalBasisMappingResource", url = "${configuration-service.url}")
|
||||
public interface LegalBasisClient extends LegalBasisMappingResource {
|
||||
}
|
||||
-102
@@ -1,102 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
public class MockMultipartFile implements MultipartFile {
|
||||
|
||||
private final String name;
|
||||
private final String originalFilename;
|
||||
@Nullable
|
||||
private final String contentType;
|
||||
private final byte[] content;
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable byte[] content) {
|
||||
|
||||
this(name, "", null, content);
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, InputStream contentStream) throws IOException {
|
||||
|
||||
this(name, "", null, FileCopyUtils.copyToByteArray(contentStream));
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType,
|
||||
@Nullable byte[] content) {
|
||||
|
||||
Assert.hasLength(name, "Name must not be empty");
|
||||
this.name = name;
|
||||
this.originalFilename = originalFilename != null ? originalFilename : "";
|
||||
this.contentType = contentType;
|
||||
this.content = content != null ? content : new byte[0];
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType,
|
||||
InputStream contentStream) throws IOException {
|
||||
|
||||
this(name, originalFilename, contentType, FileCopyUtils.copyToByteArray(contentStream));
|
||||
}
|
||||
|
||||
|
||||
public String getName() {
|
||||
|
||||
return this.name;
|
||||
}
|
||||
|
||||
|
||||
@NonNull
|
||||
public String getOriginalFilename() {
|
||||
|
||||
return this.originalFilename;
|
||||
}
|
||||
|
||||
|
||||
@Nullable
|
||||
public String getContentType() {
|
||||
|
||||
return this.contentType;
|
||||
}
|
||||
|
||||
|
||||
public boolean isEmpty() {
|
||||
|
||||
return this.content.length == 0;
|
||||
}
|
||||
|
||||
|
||||
public long getSize() {
|
||||
|
||||
return this.content.length;
|
||||
}
|
||||
|
||||
|
||||
public byte[] getBytes() throws IOException {
|
||||
|
||||
return this.content;
|
||||
}
|
||||
|
||||
|
||||
public InputStream getInputStream() throws IOException {
|
||||
|
||||
return new ByteArrayInputStream(this.content);
|
||||
}
|
||||
|
||||
|
||||
public void transferTo(File dest) throws IOException, IllegalStateException {
|
||||
|
||||
FileCopyUtils.copy(this.content, dest);
|
||||
}
|
||||
|
||||
}
|
||||
+4
-3
@@ -1,8 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
@FeignClient(name = "RulesResource", url = "${configuration-service.url}")
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface RulesClient extends RulesResource {
|
||||
}
|
||||
}
|
||||
+6
-4
@@ -1,15 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import com.iqser.red.commons.spring.ErrorMessage;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestControllerAdvice
|
||||
@@ -36,4 +38,4 @@ public class ControllerAdvice {
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+86
-95
@@ -1,172 +1,163 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.file.management.v1.api.model.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final PdfVisualisationService pdfVisualisationService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final DictionaryService dictionaryService;
|
||||
private final AnnotationService annotationService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
|
||||
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
|
||||
var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
|
||||
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
|
||||
annotationService.annotate(pdDocument, redactionLog, sectionsGrid);
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
pdDocument.save(byteArrayOutputStream);
|
||||
return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
}
|
||||
private final AnnotationHighlightService annotationHighlightService;
|
||||
private final EntityRedactionService entityRedactionService;
|
||||
private final PdfFlattenService pdfFlattenService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest redactionRequest) {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
|
||||
annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
|
||||
.getManualRedactions());
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
if (redactionRequest.isFlatRedaction()) {
|
||||
PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
|
||||
return convert(flatDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
}
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
Document classifiedDoc;
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
try {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true);
|
||||
} catch (Exception e) {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Page page : classifiedDoc.getPages()) {
|
||||
for (AbstractTextContainer textContainer : page.getTextBlocks()) {
|
||||
if (textContainer instanceof Table) {
|
||||
Table table = (Table) textContainer;
|
||||
sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RedactionResult.builder().document(sb.toString().getBytes()).build();
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Page page : classifiedDoc.getPages()) {
|
||||
for (AbstractTextContainer textContainer : page.getTextBlocks()) {
|
||||
if (textContainer instanceof Table) {
|
||||
Table table = (Table) textContainer;
|
||||
sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RedactionResult.builder().document(sb.toString().getBytes()).build();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void updateRules(@PathVariable(RULE_SET_PARAMETER_NAME) String dossierTemplateId) {
|
||||
public void updateRules(@RequestBody String rules) {
|
||||
|
||||
droolsExecutionService.updateRules(dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void testRules(@RequestBody String rules) {
|
||||
|
||||
droolsExecutionService.testRules(rules);
|
||||
droolsExecutionService.updateRules(rules);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
|
||||
|
||||
return convert(document, numberOfPages, null, null, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages,
|
||||
List<RedactionLogEntry> redactionLogEntities,
|
||||
SectionGrid sectionGrid, long dictionaryVersion, long rulesVersion) throws IOException {
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
document.save(byteArrayOutputStream);
|
||||
return RedactionResult.builder()
|
||||
.document(byteArrayOutputStream.toByteArray())
|
||||
.numberOfPages(numberOfPages)
|
||||
.redactionLog(new RedactionLog(redactionLogEntities,dictionaryVersion, rulesVersion))
|
||||
.sectionGrid(sectionGrid)
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.memory;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.text.CharacterIterator;
|
||||
import java.text.StringCharacterIterator;
|
||||
|
||||
@Slf4j
|
||||
public class MemoryStats {
|
||||
|
||||
|
||||
public static void printMemoryStats() {
|
||||
log.info("\n\n ------------------------------ \n" +
|
||||
" Used Memory: " + humanReadableByteCountBin(getUsedMemory()) + "\n" +
|
||||
" Free Memory: " + humanReadableByteCountBin(getFreeMemory()) + "\n" +
|
||||
" Total Memory: " + humanReadableByteCountBin(getTotalMemory()) + "\n" +
|
||||
" Max Memory: " + humanReadableByteCountBin(getMaxMemory()) + "\n" +
|
||||
"\n ------------------------------ \n");
|
||||
}
|
||||
|
||||
|
||||
public static String humanReadableByteCountBin(long bytes) {
|
||||
long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
|
||||
if (absB < 1024) {
|
||||
return bytes + " B";
|
||||
}
|
||||
long value = absB;
|
||||
CharacterIterator ci = new StringCharacterIterator("KMGTPE");
|
||||
for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) {
|
||||
value >>= 10;
|
||||
ci.next();
|
||||
}
|
||||
value *= Long.signum(bytes);
|
||||
return String.format("%.1f %ciB", value / 1024.0, ci.current());
|
||||
}
|
||||
|
||||
private static long getMaxMemory() {
|
||||
return Runtime.getRuntime().maxMemory();
|
||||
}
|
||||
|
||||
private static long getUsedMemory() {
|
||||
return getMaxMemory() - getFreeMemory();
|
||||
}
|
||||
|
||||
private static long getTotalMemory() {
|
||||
return Runtime.getRuntime().totalMemory();
|
||||
}
|
||||
|
||||
private static long getFreeMemory() {
|
||||
return Runtime.getRuntime().freeMemory();
|
||||
}
|
||||
}
|
||||
-92
@@ -1,92 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class PDFAreaTextStripper extends PDFTextStripperByArea {
|
||||
|
||||
@Getter
|
||||
private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
public PDFAreaTextStripper() throws IOException {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0"))) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
|
||||
if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")) && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
|
||||
public void clearPositions() {
|
||||
textPositionSequences = new ArrayList<>();
|
||||
}
|
||||
|
||||
}
|
||||
+72
-169
@@ -1,55 +1,61 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.reflect.FieldUtils;
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.contentstream.operator.OperatorName;
|
||||
import org.apache.pdfbox.contentstream.operator.color.*;
|
||||
import org.apache.pdfbox.contentstream.operator.state.*;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.contentstream.operator.OperatorName;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
@Getter
|
||||
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
@Getter
|
||||
private final List<Ruling> rulings = new ArrayList<>();
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
@Setter
|
||||
protected PDPage pdpage;
|
||||
@Getter
|
||||
private int minCharWidth;
|
||||
@Getter
|
||||
private int maxCharWidth;
|
||||
@Getter
|
||||
private int minCharHeight;
|
||||
private int maxCharWidths;
|
||||
|
||||
@Getter
|
||||
private int maxCharHeight;
|
||||
|
||||
@Getter
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private final List<Ruling> rulings = new ArrayList<>();
|
||||
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
|
||||
private float path_x;
|
||||
private float path_y;
|
||||
@@ -57,9 +63,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
|
||||
public PDFLinesTextStripper() throws IOException {
|
||||
|
||||
super();
|
||||
this.addOperator(new SetStrokingColorSpace());
|
||||
this.addOperator(new SetNonStrokingColorSpace());
|
||||
@@ -83,9 +87,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
this.addOperator(new SetLineWidth());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments)
|
||||
throws IOException {
|
||||
|
||||
String operation = operator.getName();
|
||||
|
||||
@@ -106,11 +110,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
// The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
|
||||
if (pos.getY() > path_y) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
|
||||
.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
|
||||
.getX(), path_y)));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
|
||||
}
|
||||
|
||||
path_x = (float) pos.getX();
|
||||
@@ -131,25 +133,19 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
Point2D p2 = transformPosition(x + width, y + height);
|
||||
|
||||
// Horizontal lines
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
|
||||
// Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
}
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -172,62 +168,12 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
case OperatorName.ENDPATH:
|
||||
graphicsPath.clear();
|
||||
break;
|
||||
|
||||
case OperatorName.DRAW_OBJECT:
|
||||
processImageOperation(arguments);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
super.processOperator(operator, arguments);
|
||||
}
|
||||
|
||||
|
||||
protected void processImageOperation(List<COSBase> arguments) {
|
||||
|
||||
try {
|
||||
COSName objectName = (COSName) arguments.get(0);
|
||||
PDXObject xobject = getResources().getXObject(objectName);
|
||||
if (xobject instanceof PDImageXObject) {
|
||||
PDImageXObject pdfImage = (PDImageXObject) xobject;
|
||||
|
||||
Rectangle2D imageBounds = calculateImagePosition(pdfImage);
|
||||
|
||||
Rectangle2D rect = new Rectangle2D.Float((float) imageBounds.getX(), (float) imageBounds.getY(), (float) imageBounds
|
||||
.getWidth(), (float) imageBounds.getHeight());
|
||||
|
||||
// Memory Hack - sofReference kills me
|
||||
FieldUtils.writeField(pdfImage, "cachedImageSubsampling", -1, true);
|
||||
|
||||
if (rect.getHeight() > 2 && rect.getWidth() > 2) {
|
||||
this.images.add(new PdfImage(pdfImage.getImage(), rect, pageNumber));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Problem during image extraction: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Rectangle2D calculateImagePosition(PDImageXObject pdfImage) throws IOException {
|
||||
|
||||
Matrix ctm = getGraphicsState().getCurrentTransformationMatrix();
|
||||
|
||||
Rectangle2D imageBounds = pdfImage.getImage().getRaster().getBounds();
|
||||
|
||||
AffineTransform imageTransform = new AffineTransform(ctm.createAffineTransform());
|
||||
imageTransform.scale(1.0 / pdfImage.getWidth(), -1.0 / pdfImage.getHeight());
|
||||
imageTransform.translate(0, -pdfImage.getHeight());
|
||||
|
||||
AffineTransform pageTransform = new AffineTransform();
|
||||
pageTransform.concatenate(imageTransform);
|
||||
|
||||
return pageTransform.createTransformedShape(imageBounds).getBounds2D();
|
||||
}
|
||||
|
||||
|
||||
private float floatValue(COSBase value) {
|
||||
|
||||
if (value instanceof COSNumber) {
|
||||
return ((COSNumber) value).floatValue();
|
||||
} else {
|
||||
@@ -235,56 +181,38 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Point2D.Float transformPosition(float x, float y) {
|
||||
|
||||
return super.transformedPoint(x, y);
|
||||
}
|
||||
|
||||
|
||||
private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {
|
||||
|
||||
try {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
|
||||
.toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor()
|
||||
.isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
rulings.addAll(path);
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " or " + getGraphicsState().getNonStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " does not support toRGB");
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if (charWidth < minCharWidth) {
|
||||
minCharWidth = charWidth;
|
||||
}
|
||||
if (charWidth > maxCharWidth) {
|
||||
maxCharWidth = charWidth;
|
||||
}
|
||||
|
||||
int charHeight = (int) textPositions.get(i).getHeightDir();
|
||||
if (charHeight < minCharHeight) {
|
||||
minCharHeight = charHeight;
|
||||
}
|
||||
if (charWidth > maxCharHeight) {
|
||||
if(charHeight > maxCharHeight){
|
||||
maxCharHeight = charHeight;
|
||||
}
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0"))) {
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if(charWidth > maxCharWidths){
|
||||
maxCharWidths = charWidth;
|
||||
}
|
||||
|
||||
if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
@@ -292,33 +220,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
|
||||
if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
|
||||
if (i > 0 && textPositions.get(i).getUnicode().equals(" ") && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")) && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i + 1;
|
||||
@@ -326,29 +236,22 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
|
||||
if (!sublist.isEmpty() && sublist.get(sublist.size() - 1).getUnicode().equals(" ")) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\u00A0")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText(PDDocument doc) throws IOException {
|
||||
|
||||
minCharWidth = Integer.MAX_VALUE;
|
||||
maxCharWidth = 0;
|
||||
minCharHeight = Integer.MAX_VALUE;
|
||||
maxCharHeight = 0;
|
||||
maxCharWidths = 0;
|
||||
maxCharWidths = 0;
|
||||
textPositionSequences.clear();
|
||||
images = new ArrayList<>();
|
||||
rulings.clear();
|
||||
graphicsPath.clear();
|
||||
path_x = 0.0f;
|
||||
@@ -358,4 +261,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class ParsedElements {
|
||||
|
||||
private List<TextPositionSequence> sequences;
|
||||
private List<Ruling> rulings;
|
||||
|
||||
private boolean landscape;
|
||||
private boolean rotated;
|
||||
|
||||
private float maxCharWidth;
|
||||
private float maxCharHeight;
|
||||
}
|
||||
-52
@@ -1,52 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class RedTextPosition {
|
||||
|
||||
private String textMatrix;
|
||||
private int rotation;
|
||||
private float y;
|
||||
private float pageHeight;
|
||||
private float pageWidth;
|
||||
private String unicode;
|
||||
private float XDirAdj;
|
||||
private float YDirAdj;
|
||||
private float width;
|
||||
private float heightDir;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
private float widthOfSpace;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
private float fontSizeInPt;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
private String fontName;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public static RedTextPosition fromTextPosition(TextPosition textPosition) {
|
||||
var pos = new RedTextPosition();
|
||||
BeanUtils.copyProperties(textPosition, pos);
|
||||
pos.setFontName(textPosition.getFont().getName());
|
||||
|
||||
pos.setFontSizeInPt(textPosition.getFontSizeInPt());
|
||||
|
||||
pos.setTextMatrix(textPosition.getTextMatrix().toString());
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+28
-69
@@ -1,52 +1,32 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@JsonIgnoreProperties({ "empty" })
|
||||
@RequiredArgsConstructor
|
||||
public class TextPositionSequence implements CharSequence {
|
||||
|
||||
private int page;
|
||||
private List<RedTextPosition> textPositions = new ArrayList<>();
|
||||
private List<TextPosition> textPositions = new ArrayList<>();
|
||||
|
||||
private float x1;
|
||||
private float x2;
|
||||
|
||||
public TextPositionSequence(int page) {
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
|
||||
public static TextPositionSequence fromData(List<RedTextPosition> textPositions, int page) {
|
||||
var textPositionSequence = new TextPositionSequence();
|
||||
textPositionSequence.textPositions = textPositions;
|
||||
textPositionSequence.page = page;
|
||||
|
||||
return textPositionSequence;
|
||||
}
|
||||
private final int page;
|
||||
|
||||
|
||||
public TextPositionSequence(List<TextPosition> textPositions, int page) {
|
||||
|
||||
this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
|
||||
this.textPositions = textPositions;
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public int length() {
|
||||
|
||||
@@ -57,7 +37,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
|
||||
RedTextPosition textPosition = textPositionAt(index);
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return text.charAt(0);
|
||||
}
|
||||
@@ -65,7 +45,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
public char charAt(int index, boolean caseInSensitive) {
|
||||
|
||||
RedTextPosition textPosition = textPositionAt(index);
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
|
||||
}
|
||||
@@ -74,7 +54,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@Override
|
||||
public TextPositionSequence subSequence(int start, int end) {
|
||||
|
||||
return fromData(textPositions.subList(start, end), page);
|
||||
return new TextPositionSequence(textPositions.subList(start, end), page);
|
||||
}
|
||||
|
||||
|
||||
@@ -89,25 +69,18 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
public RedTextPosition textPositionAt(int index) {
|
||||
public TextPosition textPositionAt(int index) {
|
||||
|
||||
return textPositions.get(index);
|
||||
}
|
||||
|
||||
|
||||
public void add(RedTextPosition textPosition) {
|
||||
public void add(TextPosition textPosition) {
|
||||
|
||||
this.textPositions.add(textPosition);
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPosition textPosition) {
|
||||
|
||||
this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getX1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
@@ -118,7 +91,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getX2() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
@@ -129,14 +101,10 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public float getRotationAdjustedY() {
|
||||
|
||||
return textPositions.get(0).getY();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getY1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
@@ -147,7 +115,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getY2() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
@@ -158,40 +125,38 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getTextHeight() {
|
||||
|
||||
return textPositions.get(0).getHeightDir() + 2;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getHeight() {
|
||||
|
||||
return getY2() - getY1();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getWidth() {
|
||||
|
||||
return getX2() - getX1();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public String getFont() {
|
||||
return textPositions.get(0).getFontName()
|
||||
|
||||
return textPositions.get(0)
|
||||
.getFont()
|
||||
.toString()
|
||||
.toLowerCase()
|
||||
.replaceAll(",bold", "")
|
||||
.replaceAll(",italic", "");
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public String getFontStyle() {
|
||||
|
||||
String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
|
||||
String lowercaseFontName = textPositions.get(0).getFont().toString().toLowerCase();
|
||||
|
||||
if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
|
||||
return "bold, italic";
|
||||
@@ -205,25 +170,25 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
public float getFontSize() {
|
||||
|
||||
return textPositions.get(0).getFontSizeInPt();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
public float getSpaceWidth() {
|
||||
|
||||
return textPositions.get(0).getWidthOfSpace();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
public int getRotation() {
|
||||
|
||||
return textPositions.get(0).getRotation();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
public Rectangle getRectangle() {
|
||||
|
||||
float height = getTextHeight();
|
||||
@@ -237,13 +202,6 @@ public class TextPositionSequence implements CharSequence {
|
||||
posXEnd = textPositions.get(0).getYDirAdj() + 2;
|
||||
posYInit = getY1();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
|
||||
} else if (textPositions.get(0).getRotation() == 270) {
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
|
||||
.getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + height;
|
||||
} else {
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
@@ -255,4 +213,5 @@ public class TextPositionSequence implements CharSequence {
|
||||
return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class MessagingConfiguration {
|
||||
|
||||
|
||||
public static final String REDACTION_QUEUE = "redactionQueue";
|
||||
|
||||
public static final String REDACTION_DQL = "redactionDQL";
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionDeadLetterQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_DQL).build();
|
||||
}
|
||||
}
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ReanalyzeService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionMessageReceiver {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ReanalyzeService reanalyzeService;
|
||||
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_QUEUE)
|
||||
public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
|
||||
log.info("Processing analyze request: {}", analyzeRequest);
|
||||
AnalyzeResult result;
|
||||
if (analyzeRequest.isReanalyseOnlyIfPossible()) {
|
||||
result = reanalyzeService.reanalyze(analyzeRequest);
|
||||
} else {
|
||||
result = reanalyzeService.analyze(analyzeRequest);
|
||||
}
|
||||
log.info("Successfully analyzed {}", analyzeRequest);
|
||||
|
||||
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
|
||||
}
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_DQL)
|
||||
public void receiveAnalyzeRequestDQL(String in) throws JsonProcessingException {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
|
||||
log.info("Failed to process analyze request: {}", analyzeRequest);
|
||||
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
|
||||
}
|
||||
+15
-28
@@ -3,46 +3,33 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import lombok.Value;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Value
|
||||
public class CellValue {
|
||||
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
TextBlock textBlock;
|
||||
|
||||
int rowSpanStart;
|
||||
|
||||
private int rowSpanStart;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
while (itty.hasNext()) {
|
||||
TextBlock textBlock = itty.next();
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
if (itty.hasNext()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
@@ -50,4 +37,4 @@ public class CellValue {
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+12
-23
@@ -1,39 +1,25 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Dictionary {
|
||||
|
||||
public static final String RECOMMENDATION_PREFIX = "recommendation_";
|
||||
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionaryModels;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
private Map<String, DictionaryModel> localAccessMap;
|
||||
|
||||
@Getter
|
||||
private DictionaryVersion version;
|
||||
|
||||
|
||||
public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
|
||||
this.dictionaryModels = dictionaryModels;
|
||||
this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type) {
|
||||
if (!localAccessMap.containsKey(type)) {
|
||||
return 0;
|
||||
}
|
||||
return localAccessMap.get(type).getRank();
|
||||
}
|
||||
private long version;
|
||||
|
||||
|
||||
public boolean isRecommendation(String type) {
|
||||
@@ -60,7 +46,7 @@ public class Dictionary {
|
||||
|
||||
public boolean containsValue(String type, String value) {
|
||||
|
||||
return localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
if (localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getLocalEntries()
|
||||
@@ -68,7 +54,10 @@ public class Dictionary {
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
|
||||
.getLocalEntries()
|
||||
.contains(value);
|
||||
.contains(value)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryIncrement {
|
||||
|
||||
private Set<DictionaryIncrementValue> values;
|
||||
private DictionaryVersion dictionaryVersion;
|
||||
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryIncrementValue {
|
||||
|
||||
private String value;
|
||||
private boolean caseinsensitive;
|
||||
|
||||
}
|
||||
+6
-11
@@ -1,17 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryModel implements Serializable {
|
||||
public class DictionaryModel {
|
||||
|
||||
private String type;
|
||||
private int rank;
|
||||
@@ -19,13 +16,11 @@ public class DictionaryModel implements Serializable {
|
||||
private boolean caseInsensitive;
|
||||
private boolean hint;
|
||||
private boolean recommendation;
|
||||
private Set<DictionaryEntry> entries;
|
||||
private Set<String> entries;
|
||||
private Set<String> localEntries;
|
||||
private boolean isDossierDictionary;
|
||||
|
||||
public Set<String> getValues(boolean local) {
|
||||
return local ? localEntries : entries.stream().filter(e -> !e.isDeleted()).map(e -> e.getValue()).collect(Collectors
|
||||
.toSet());
|
||||
public Set<String> getValues(boolean local){
|
||||
return local ? localEntries : entries;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
public class DictionaryRepresentation {
|
||||
|
||||
private String dossierTemplateId;
|
||||
private long dictionaryVersion = -1;
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
private float[] defaultColor;
|
||||
private float[] requestAddColor;
|
||||
private float[] requestRemoveColor;
|
||||
private float[] notRedactedColor;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DictionaryVersion {
|
||||
|
||||
long dossierTemplateVersion;
|
||||
long dossierVersion;
|
||||
}
|
||||
+8
-25
@@ -1,17 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity {
|
||||
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private final String word;
|
||||
private final String type;
|
||||
private boolean redaction;
|
||||
@@ -19,10 +20,7 @@ public class Entity {
|
||||
private String legalBasis;
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer end;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
@@ -32,15 +30,8 @@ public class Entity {
|
||||
@EqualsAndHashCode.Include
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end, boolean isDossierDictionaryEntry) {
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -51,16 +42,10 @@ public class Entity {
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
this.textAfter = textAfter;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
}
|
||||
|
||||
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry) {
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -68,8 +53,6 @@ public class Entity {
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-4
@@ -1,23 +1,24 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class EntityPositionSequence {
|
||||
|
||||
private final String id;
|
||||
@EqualsAndHashCode.Exclude
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
private int pageNumber;
|
||||
private final String id;
|
||||
|
||||
}
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Image {
|
||||
|
||||
private String type;
|
||||
private RedRectangle2D position;
|
||||
private boolean redaction;
|
||||
private String redactionReason;
|
||||
private String legalBasis;
|
||||
private int matchedRule;
|
||||
private int sectionNumber;
|
||||
private String section;
|
||||
private int page;
|
||||
|
||||
}
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
public enum ImageType {
|
||||
LOGO, FORMULA, SIGNATURE, OTHER, OCR
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
|
||||
@Data
|
||||
@RequiredArgsConstructor
|
||||
public class PdfImage {
|
||||
|
||||
@JsonIgnore
|
||||
private BufferedImage image;
|
||||
@NonNull
|
||||
private RedRectangle2D position;
|
||||
private ImageType imageType;
|
||||
private boolean isAppendedToParagraph;
|
||||
|
||||
@NonNull
|
||||
private int page;
|
||||
|
||||
public PdfImage(BufferedImage image, Rectangle2D position, int page) {
|
||||
this.image = image;
|
||||
this.position = new RedRectangle2D(position.getX(), position.getY(), position.getWidth(), position.getHeight());
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedRectangle2D {
|
||||
|
||||
public static final double THRESHOLD = 0.01;
|
||||
|
||||
private double x;
|
||||
private double y;
|
||||
private double width;
|
||||
private double height;
|
||||
|
||||
@JsonIgnore
|
||||
public boolean isEmpty() {
|
||||
return width <= 0.0f || height <= 0.0f;
|
||||
}
|
||||
|
||||
public boolean contains(double x, double y, double w, double h) {
|
||||
if (isEmpty() || w <= 0 || h <= 0) {
|
||||
return false;
|
||||
}
|
||||
double x0 = getX();
|
||||
double y0 = getY();
|
||||
return round(x) >= round(x0) &&
|
||||
round(y) >= round(y0) &&
|
||||
(x + w) - (x0 + getWidth()) <= THRESHOLD &&
|
||||
(y + h) - (y0 + getHeight()) <= THRESHOLD;
|
||||
}
|
||||
|
||||
private double round(double value) {
|
||||
double d = Math.pow(10, 2);
|
||||
return Math.round(value * d) / d;
|
||||
}
|
||||
}
|
||||
+17
-34
@@ -1,14 +1,14 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
public class SearchableText {
|
||||
|
||||
private final List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
@@ -32,7 +32,6 @@ public class SearchableText {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("checkstyle:ModifiedControlVariable")
|
||||
public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive,
|
||||
List<TextPositionSequence> sequencesSubList) {
|
||||
@@ -67,12 +66,9 @@ public class SearchableText {
|
||||
for (int j = 0; j < searchSpace.get(i).length(); j++) {
|
||||
|
||||
if (i > 0 && j == 0 && searchSpace.get(i).charAt(0, caseInsensitive) == ' ' && searchSpace.get(i - 1)
|
||||
.charAt(searchSpace.get(i - 1)
|
||||
.length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i)
|
||||
.charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions()
|
||||
.isEmpty()) {
|
||||
.charAt(searchSpace.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == ' ' && searchSpace.get(i).charAt(j - 1, caseInsensitive) == ' ') {
|
||||
if (j == searchSpace.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
}
|
||||
continue;
|
||||
@@ -84,8 +80,8 @@ public class SearchableText {
|
||||
counter++;
|
||||
}
|
||||
|
||||
if (searchSpace.get(i).charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace
|
||||
.get(i)
|
||||
if (searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) == '-') {
|
||||
|
||||
if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(searchSpace.get(i)
|
||||
@@ -104,15 +100,14 @@ public class SearchableText {
|
||||
if (counter == searchString.length()) {
|
||||
crossSequenceParts.add(partMatch);
|
||||
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i)
|
||||
.length() - 1 || j != searchSpace.get(i).length() - 1 && isSeparator(searchSpace.get(i)
|
||||
if (i == searchSpace.size() - 1 && j == searchSpace.get(i).length() - 1 || j != searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i)
|
||||
.charAt(j + 1, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && isSeparator(searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i)
|
||||
.length() - 1 && searchSpace.get(i)
|
||||
.charAt(0, caseInsensitive)) || j == searchSpace.get(i).length() - 1 && searchSpace.get(i)
|
||||
.charAt(j, caseInsensitive) != ' ' && searchSpace.get(i + 1)
|
||||
.charAt(0, caseInsensitive) != ' ') {
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts, normalizedSearchString));
|
||||
finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
|
||||
}
|
||||
|
||||
counter = 0;
|
||||
@@ -135,21 +130,15 @@ public class SearchableText {
|
||||
}
|
||||
|
||||
return finalMatches;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts,
|
||||
String searchString) {
|
||||
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
String asString = buildString(crossSequenceParts);
|
||||
if (!asString.equalsIgnoreCase(searchString)) {
|
||||
return result;
|
||||
}
|
||||
private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
|
||||
|
||||
String plainId = IdBuilder.buildId(crossSequenceParts);
|
||||
String id = plainId;
|
||||
|
||||
List<EntityPositionSequence> result = new ArrayList<>();
|
||||
int currentPage = -1;
|
||||
int idDiffentPageSuffix = 1;
|
||||
EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
|
||||
@@ -184,12 +173,6 @@ public class SearchableText {
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return buildString(sequences);
|
||||
}
|
||||
|
||||
|
||||
public String buildString(List<TextPositionSequence> sequences) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
TextPositionSequence previous = null;
|
||||
@@ -232,4 +215,4 @@ public class SearchableText {
|
||||
return sb.append("\n").toString();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+110
-261
@@ -1,23 +1,23 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary.RECOMMENDATION_PREFIX;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Data
|
||||
@Slf4j
|
||||
@@ -45,96 +45,36 @@ public class Section {
|
||||
|
||||
private Map<String, CellValue> tabularData;
|
||||
|
||||
private Dictionary dictionary;
|
||||
|
||||
private SearchableText searchableText;
|
||||
|
||||
@Builder.Default
|
||||
private Set<Image> images = new HashSet<>();
|
||||
|
||||
|
||||
public boolean rowEquals(String headerName, String value) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName)
|
||||
.toString()
|
||||
.getTextBlock()
|
||||
.getText()
|
||||
.equals(value);
|
||||
}
|
||||
|
||||
|
||||
public boolean hasTableHeader(String headerName) {
|
||||
|
||||
String cleanHeaderName = headerName.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
|
||||
return tabularData != null && tabularData.containsKey(cleanHeaderName);
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesType(String type) {
|
||||
|
||||
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesImageType(String type) {
|
||||
|
||||
return images.stream().anyMatch(image -> image.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
public boolean headlineContainsWord(String word) {
|
||||
|
||||
return StringUtils.containsIgnoreCase(headline, word);
|
||||
}
|
||||
|
||||
|
||||
public void expandByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Set<Entity> expanded = new HashSet<>();
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity
|
||||
.getRedactionReason(), entity.getLegalBasis()));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
|
||||
|
||||
public void redactImage(String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
images.forEach(image -> {
|
||||
if (image.getType().equals(type)) {
|
||||
image.setRedaction(true);
|
||||
image.setMatchedRule(ruleNumber);
|
||||
image.setRedactionReason(reason);
|
||||
image.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void redact(String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
boolean hasRecommendactionDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
if (entity.getType().equals(type) || hasRecommendactionDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
@@ -145,24 +85,12 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
public void redactNotImage(String type, int ruleNumber, String reason) {
|
||||
|
||||
images.forEach(image -> {
|
||||
if (image.getType().equals(type)) {
|
||||
image.setRedaction(false);
|
||||
image.setMatchedRule(ruleNumber);
|
||||
image.setRedactionReason(reason);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public void redactNot(String type, int ruleNumber, String reason) {
|
||||
|
||||
boolean hasRecommendationDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
boolean hasRecommendactionDictionary = dictionaryTypes.contains(RECOMMENDATION_PREFIX + type);
|
||||
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(type) || hasRecommendationDictionary && entity.getType()
|
||||
if (entity.getType().equals(type) || hasRecommendactionDictionary && entity.getType()
|
||||
.equals(RECOMMENDATION_PREFIX + type)) {
|
||||
entity.setRedaction(false);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
@@ -172,49 +100,6 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group,
|
||||
String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Set<Entity> expanded = new HashSet<>();
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (!entity.getType().equals(type) || entity.getTextAfter() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(entity.getTextAfter());
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, expanded, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
|
||||
|
||||
public void addHintAnnotationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
entities.forEach(entity -> {
|
||||
@@ -230,15 +115,8 @@ public class Section {
|
||||
|
||||
public void addHintAnnotation(String value, String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
|
||||
public void addRedaction(String value, String asType, int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true);
|
||||
entities.addAll(found);
|
||||
}
|
||||
|
||||
|
||||
@@ -250,88 +128,24 @@ public class Section {
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void recommendLineAfter(String start, String asType) {
|
||||
|
||||
String[] values = StringUtils.substringsBetween(text, start, "\n");
|
||||
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
|
||||
String trimmedValue = value.trim();
|
||||
String cleanValue;
|
||||
if (trimmedValue.startsWith(":")) {
|
||||
cleanValue = trimmedValue.substring(1).trim();
|
||||
} else {
|
||||
cleanValue = trimmedValue;
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>())
|
||||
.add(cleanValue);
|
||||
}
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
public void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
|
||||
String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addRecommendationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(text);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
|
||||
|
||||
Matcher matcher = compiledPattern.matcher(searchText);
|
||||
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -343,16 +157,26 @@ public class Section {
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -365,16 +189,12 @@ public class Section {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
String[] lines = value.split("\n");
|
||||
|
||||
for (String line : lines) {
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false);
|
||||
|
||||
if (line.trim().length() <= 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
@@ -382,6 +202,61 @@ public class Section {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO No need to iterate
|
||||
entities.forEach(entity -> {
|
||||
if (entity.getType().equals(asType)) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseinsensitive) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
String text = caseinsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseinsensitive ? value.toLowerCase() : value;
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = text.indexOf(searchValue, stopIndex);
|
||||
stopIndex = startIndex + searchValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(text.charAt(startIndex - 1)) || isSeparator(text
|
||||
.charAt(startIndex - 1))) && (stopIndex == text.length() || isSeparator(text.charAt(stopIndex)))) {
|
||||
found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
|
||||
return removeEntitiesContainedInLarger(found);
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
@@ -405,27 +280,6 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseInsensitive ? value.toLowerCase() : value;
|
||||
|
||||
Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, true, false);
|
||||
|
||||
found.forEach(entity -> {
|
||||
if (redacted) {
|
||||
entity.setRedaction(true);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setLegalBasis(legalBasis);
|
||||
}
|
||||
});
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact,
|
||||
boolean addAsRecommendations, String reason, String legalBasis) {
|
||||
|
||||
@@ -437,24 +291,19 @@ public class Section {
|
||||
} else {
|
||||
String word = value.toString();
|
||||
|
||||
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber, false, false);
|
||||
Entity entity = new Entity(word, type, value.getRowSpanStart(), value.getRowSpanStart() + word.length(), headline, sectionNumber);
|
||||
entity.setRedaction(redact);
|
||||
entity.setMatchedRule(ruleNumber);
|
||||
entity.setRedactionReason(reason);
|
||||
entity.setTargetSequences(value.getTextBlocks()
|
||||
.stream()
|
||||
.map(TextBlock::getSequences)
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.toList())); // Make sure no other cells with same content are highlighted
|
||||
entity.setTargetSequences(value.getTextBlock()
|
||||
.getSequences()); // Make sure no other cells with same content are highlighted
|
||||
entity.setLegalBasis(legalBasis);
|
||||
|
||||
Set<Entity> singleEntitySet = new HashSet<>();
|
||||
singleEntitySet.add(entity);
|
||||
EntitySearchUtils.clearAndFindPositions(singleEntitySet, searchableText, dictionary);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
entities.remove(entity);
|
||||
entities.add(entity);
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, entity, dictionary);
|
||||
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
entities = removeEntitiesContainedInLarger(entities);
|
||||
|
||||
if (addAsRecommendations && !isLocal()) {
|
||||
String cleanedWord = word.replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
|
||||
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
public class AnalyzeResponseService {
|
||||
|
||||
public AnalyzeResult createAnalyzeResponse(String dossierId, String fileId, long duration, int pageCount, RedactionLog redactionLog, RedactionChangeLog redactionChangeLog) {
|
||||
boolean hasHints = redactionLog.getRedactionLogEntry().stream().anyMatch(RedactionLogEntry::isHint);
|
||||
|
||||
boolean hasRequests = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.anyMatch(entry -> entry.isManual() && entry.getStatus()
|
||||
.equals(com.iqser.red.service.redaction.v1.model.Status.REQUESTED));
|
||||
|
||||
boolean hasRedactions = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.anyMatch(entry -> entry.isRedacted() && !entry.isManual() || entry.isManual() && entry.getStatus()
|
||||
.equals(com.iqser.red.service.redaction.v1.model.Status.APPROVED));
|
||||
|
||||
boolean hasImages = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.anyMatch(entry -> entry.isHint() && entry.getType().equals("image"));
|
||||
|
||||
boolean hasUpdates = redactionChangeLog != null && redactionChangeLog.getRedactionLogEntry() != null && !redactionChangeLog
|
||||
.getRedactionLogEntry()
|
||||
.isEmpty() && redactionChangeLog.getRedactionLogEntry().stream().anyMatch(entry -> !entry.getType().equals("false_positive"));
|
||||
|
||||
return AnalyzeResult.builder()
|
||||
.dossierId(dossierId)
|
||||
.fileId(fileId)
|
||||
.duration(duration)
|
||||
.numberOfPages(pageCount)
|
||||
.hasHints(hasHints)
|
||||
.hasRedactions(hasRedactions)
|
||||
.hasRequests(hasRequests)
|
||||
.hasImages(hasImages)
|
||||
.hasUpdates(hasUpdates)
|
||||
.rulesVersion(redactionLog.getRulesVersion())
|
||||
.dictionaryVersion(redactionLog.getDictionaryVersion())
|
||||
.legalBasisVersion(redactionLog.getLegalBasisVersion())
|
||||
.dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
-260
@@ -1,260 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.PDPageContentStream;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
|
||||
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class AnnotationService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
|
||||
List<SectionRectangle> sectionRectangles = sectionGrid.getRectanglesPerPage().get(page);
|
||||
if (sectionRectangles != null && !sectionRectangles.isEmpty()) {
|
||||
drawSectionGrid(document, pdPage, sectionRectangles);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
|
||||
if (logEntries != null && !logEntries.isEmpty()) {
|
||||
addAnnotations(logEntries, pdPage, page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {
|
||||
|
||||
List<PDAnnotation> annotations = pdPage.getAnnotations();
|
||||
|
||||
for (RedactionLogEntry entry : logEntries) {
|
||||
annotations.addAll(createAnnotation(entry, page, pdPage.getMediaBox(), pdPage.getCropBox()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, PDRectangle mediaBox,
|
||||
PDRectangle cropBox) {
|
||||
|
||||
List<PDAnnotation> annotations = new ArrayList<>();
|
||||
|
||||
List<Rectangle> rectangles = redactionLogEntry.getPositions()
|
||||
.stream()
|
||||
.filter(pos -> pos.getPage() == page)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (rectangles.isEmpty()) {
|
||||
return annotations;
|
||||
}
|
||||
|
||||
PDAnnotationTextMarkup annotation = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
|
||||
annotation.constructAppearances();
|
||||
PDRectangle pdRectangle = toPDRectangle(rectangles, mediaBox, cropBox);
|
||||
annotation.setRectangle(pdRectangle);
|
||||
annotation.setQuadPoints(toQuadPoints(rectangles, mediaBox, cropBox));
|
||||
if (!redactionLogEntry.isHint()) {
|
||||
annotation.setContents(createAnnotationContent(redactionLogEntry));
|
||||
}
|
||||
annotation.setTitlePopup(redactionLogEntry.getId());
|
||||
annotation.setAnnotationName(redactionLogEntry.getId());
|
||||
annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
|
||||
annotations.add(annotation);
|
||||
|
||||
if (redactionLogEntry.getComments() != null) {
|
||||
for (Comment comment : redactionLogEntry.getComments()) {
|
||||
PDAnnotationText txtAnnot = new PDAnnotationText();
|
||||
txtAnnot.setAnnotationName(comment.getId());
|
||||
txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
|
||||
txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
|
||||
txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));
|
||||
txtAnnot.setTitlePopup(comment.getUser());
|
||||
txtAnnot.setContents(comment.getText());
|
||||
txtAnnot.setRectangle(pdRectangle);
|
||||
annotations.add(txtAnnot);
|
||||
}
|
||||
}
|
||||
|
||||
return annotations;
|
||||
}
|
||||
|
||||
|
||||
private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.isManual()) {
|
||||
return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
return "\nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
|
||||
.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
|
||||
|
||||
private PDRectangle toPDRectangle(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float lowerLeftX = Float.MAX_VALUE;
|
||||
float upperRightX = 0;
|
||||
float lowerLeftY = 0;
|
||||
float upperRightY = Float.MAX_VALUE;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
if (rectangle.getTopLeft().getX() < lowerLeftX) {
|
||||
lowerLeftX = rectangle.getTopLeft().getX();
|
||||
}
|
||||
if (rectangle.getTopLeft().getX() + rectangle.getWidth() > upperRightX) {
|
||||
upperRightX = rectangle.getTopLeft().getX() + rectangle.getWidth();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() + rectangle.getHeight() > lowerLeftY) {
|
||||
lowerLeftY = rectangle.getTopLeft().getY() + rectangle.getHeight();
|
||||
}
|
||||
if (rectangle.getTopLeft().getY() < upperRightY) {
|
||||
upperRightY = rectangle.getTopLeft().getY();
|
||||
}
|
||||
}
|
||||
|
||||
var x1 = lowerLeftX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = lowerLeftY + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = upperRightX + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = upperRightY - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
PDRectangle annotationPosition = new PDRectangle();
|
||||
annotationPosition.setLowerLeftX(x1);
|
||||
annotationPosition.setLowerLeftY(y1);
|
||||
annotationPosition.setUpperRightX(x2);
|
||||
annotationPosition.setUpperRightY(y2);
|
||||
return annotationPosition;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoints(List<Rectangle> rectangles, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
float[] quadPoints = new float[rectangles.size() * 8];
|
||||
int i = 0;
|
||||
|
||||
for (Rectangle rectangle : rectangles) {
|
||||
float[] quadPoint = toQuadPoint(rectangle, mediaBox, cropBox);
|
||||
for (int j = 0; j <= 7; j++) {
|
||||
quadPoints[i + j] = quadPoint[j];
|
||||
}
|
||||
i += 8;
|
||||
}
|
||||
return quadPoints;
|
||||
}
|
||||
|
||||
|
||||
private float[] toQuadPoint(Rectangle rectangle, PDRectangle mediaBox, PDRectangle cropBox) {
|
||||
|
||||
var x1 = rectangle.getTopLeft().getX() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y1 = rectangle.getTopLeft().getY() + (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
var x2 = rectangle.getTopLeft()
|
||||
.getX() + rectangle.getWidth() + cropBox.getLowerLeftX() - mediaBox.getLowerLeftY();
|
||||
var y2 = rectangle.getTopLeft()
|
||||
.getY() + rectangle.getHeight() - (mediaBox.getLowerLeftY() - cropBox.getLowerLeftY());
|
||||
|
||||
// quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
|
||||
// of the area to be highlighted
|
||||
return new float[]{x1, y1, x2, y2, x1, y2 - rectangle.getHeight(), x2, y1 - rectangle.getHeight()};
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionGrid(PDDocument document, PDPage pdPage,
|
||||
List<SectionRectangle> sectionRectangles) throws IOException {
|
||||
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
for (SectionRectangle sectionRectangle : sectionRectangles) {
|
||||
drawSectionRectangle(contentStream, sectionRectangle);
|
||||
drawSectionPartNumberText(contentStream, sectionRectangle);
|
||||
drawTableCells(contentStream, sectionRectangle);
|
||||
}
|
||||
contentStream.close();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionRectangle(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.LIGHT_GRAY);
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.addRect(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY(), sectionRectangle.getWidth(), sectionRectangle.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
|
||||
|
||||
private void drawSectionPartNumberText(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
contentStream.beginText();
|
||||
contentStream.setNonStrokingColor(Color.DARK_GRAY);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 8f);
|
||||
if (sectionRectangle.getTableCells() == null) {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft()
|
||||
.getY() + sectionRectangle.getHeight());
|
||||
} else {
|
||||
contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY());
|
||||
}
|
||||
contentStream.showText(sectionRectangle.getPart() + "/" + sectionRectangle.getNumberOfParts());
|
||||
contentStream.endText();
|
||||
}
|
||||
|
||||
|
||||
private void drawTableCells(PDPageContentStream contentStream,
|
||||
SectionRectangle sectionRectangle) throws IOException {
|
||||
|
||||
if (sectionRectangle.getTableCells() != null) {
|
||||
for (CellRectangle cell : sectionRectangle.getTableCells()) {
|
||||
contentStream.setLineWidth(0.5f);
|
||||
contentStream.setStrokingColor(Color.CYAN);
|
||||
contentStream.addRect(cell.getTopLeft().getX(), cell.getTopLeft()
|
||||
.getY(), cell.getWidth(), cell.getHeight());
|
||||
contentStream.stroke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Map<Integer, List<RedactionLogEntry>> convertRedactionLog(RedactionLog redactionLog) {
|
||||
|
||||
Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = new HashMap<>();
|
||||
if (redactionLog == null) {
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
int page = 0;
|
||||
for (Rectangle position : entry.getPositions()) {
|
||||
if (position.getPage() != page) {
|
||||
redactionLogPerPage.computeIfAbsent(position.getPage(), x -> new ArrayList<>()).add(entry);
|
||||
page = position.getPage();
|
||||
}
|
||||
}
|
||||
}
|
||||
return redactionLogPerPage;
|
||||
}
|
||||
|
||||
}
|
||||
+80
-139
@@ -1,115 +1,92 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.Colors;
|
||||
import com.iqser.red.service.configuration.v1.api.model.DictionaryEntry;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
|
||||
import com.iqser.red.service.configuration.v1.api.model.TypeResult;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.configuration.v1.api.resource.DictionaryResource.GLOBAL_DOSSIER;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class DictionaryService {
|
||||
|
||||
|
||||
private final DictionaryClient dictionaryClient;
|
||||
|
||||
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
|
||||
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
|
||||
@Getter
|
||||
private long dictionaryVersion = -1;
|
||||
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private float[] defaultColor;
|
||||
|
||||
@Getter
|
||||
private float[] requestAddColor;
|
||||
|
||||
@Getter
|
||||
private float[] requestRemoveColor;
|
||||
|
||||
@Getter
|
||||
private float[] notRedactedColor;
|
||||
|
||||
|
||||
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
|
||||
var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
|
||||
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, GLOBAL_DOSSIER);
|
||||
|
||||
|
||||
public void updateDictionary() {
|
||||
|
||||
long version = dictionaryClient.getVersion();
|
||||
if (version > dictionaryVersion) {
|
||||
dictionaryVersion = version;
|
||||
updateDictionaryEntry();
|
||||
}
|
||||
|
||||
long dossierDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId, dossierId);
|
||||
var dossierDictionary = dictionariesByDossier.get(dossierId);
|
||||
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
|
||||
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
|
||||
}
|
||||
|
||||
return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
|
||||
}
|
||||
|
||||
|
||||
public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
|
||||
|
||||
DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
|
||||
|
||||
Set<DictionaryIncrementValue> newValues = new HashSet<>();
|
||||
List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
|
||||
dictionaryModels.forEach(dictionaryModel -> {
|
||||
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
|
||||
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
|
||||
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
if(dictionariesByDossier.containsKey(dossierId)) {
|
||||
dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
|
||||
dictionaryModels.forEach(dictionaryModel -> {
|
||||
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
|
||||
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
|
||||
newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return new DictionaryIncrement(newValues, version);
|
||||
}
|
||||
|
||||
|
||||
private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {
|
||||
private void updateDictionaryEntry() {
|
||||
|
||||
try {
|
||||
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
|
||||
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes(dossierTemplateId, dossierId);
|
||||
TypeResponse typeResponse = dictionaryClient.getAllTypes();
|
||||
if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
|
||||
|
||||
List<DictionaryModel> dictionary = typeResponse.getTypes()
|
||||
dictionary = typeResponse.getTypes()
|
||||
.stream()
|
||||
.map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
|
||||
.isHint(), t.isRecommendation(), convertEntries(t, dossierId), new HashSet<>(), !dossierId.equals(GLOBAL_DOSSIER)))
|
||||
.isHint(), t.isRecommendation(), convertEntries(t), new HashSet<>()))
|
||||
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
|
||||
.collect(Collectors.toList());
|
||||
|
||||
dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
|
||||
localAccessMap.clear();
|
||||
dictionary.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
|
||||
Colors colors = dictionaryClient.getColors(dossierTemplateId);
|
||||
|
||||
dictionaryRepresentation.setDefaultColor(convertColor(colors.getDefaultColor()));
|
||||
dictionaryRepresentation.setRequestAddColor(convertColor(colors.getRequestAdd()));
|
||||
dictionaryRepresentation.setRequestRemoveColor(convertColor(colors.getRequestRemove()));
|
||||
dictionaryRepresentation.setNotRedactedColor(convertColor(colors.getNotRedacted()));
|
||||
dictionaryRepresentation.setDossierTemplateId(dossierTemplateId);
|
||||
dictionaryRepresentation.setDictionaryVersion(version);
|
||||
dictionaryRepresentation.setDictionary(dictionary);
|
||||
|
||||
if(dossierId.equals(GLOBAL_DOSSIER)) {
|
||||
dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
|
||||
} else {
|
||||
dictionariesByDossier.put(dossierId, dictionaryRepresentation);
|
||||
}
|
||||
Colors colors = dictionaryClient.getColors();
|
||||
defaultColor = convertColor(colors.getDefaultColor());
|
||||
requestAddColor = convertColor(colors.getRequestAdd());
|
||||
requestRemoveColor = convertColor(colors.getRequestRemove());
|
||||
notRedactedColor = convertColor(colors.getNotRedacted());
|
||||
}
|
||||
} catch (FeignException e) {
|
||||
log.warn("Got some unknown feignException", e);
|
||||
@@ -118,29 +95,30 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public void updateExternalDictionary(Dictionary dictionary, String dossierTemplateId) {
|
||||
|
||||
public void updateExternalDictionary(Dictionary dictionary){
|
||||
dictionary.getDictionaryModels().forEach(dm -> {
|
||||
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
|
||||
dictionaryClient.addEntries(dm.getType(), dossierTemplateId, new ArrayList<>(dm.getLocalEntries()), false, GLOBAL_DOSSIER);
|
||||
long externalVersion = dictionaryClient.getVersion(dossierTemplateId, GLOBAL_DOSSIER);
|
||||
if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
|
||||
dictionary.getVersion().setDossierTemplateVersion(externalVersion);
|
||||
if(dm.isRecommendation() && !dm.getLocalEntries().isEmpty()){
|
||||
dictionaryClient.addEntries(dm.getType(), new ArrayList<>(dm.getLocalEntries()), false);
|
||||
long externalVersion = dictionaryClient.getVersion();
|
||||
if(externalVersion == dictionary.getVersion() + 1){
|
||||
dictionary.setVersion(externalVersion);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private Set<DictionaryEntry> convertEntries(TypeResult t, String dossierId) {
|
||||
|
||||
Set<DictionaryEntry> entries = new HashSet<>(dictionaryClient.getDictionaryForType(t.getType(), t.getDossierTemplateId(), dossierId)
|
||||
.getEntries());
|
||||
private Set<String> convertEntries(TypeResult t) {
|
||||
|
||||
if (t.isCaseInsensitive()) {
|
||||
entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
|
||||
return dictionaryClient.getDictionaryForType(t.getType())
|
||||
.getEntries()
|
||||
.stream()
|
||||
.map(String::toLowerCase)
|
||||
.collect(Collectors.toSet());
|
||||
} else {
|
||||
return new HashSet<>(dictionaryClient.getDictionaryForType(t.getType()).getEntries());
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
|
||||
@@ -151,9 +129,9 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public boolean isCaseInsensitiveDictionary(String type, String dossierTemplateId) {
|
||||
public boolean isCaseInsensitiveDictionary(String type) {
|
||||
|
||||
DictionaryModel dictionaryModel = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type);
|
||||
if (dictionaryModel != null) {
|
||||
return dictionaryModel.isCaseInsensitive();
|
||||
}
|
||||
@@ -161,29 +139,28 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public float[] getColor(String type, String dossierTemplateId) {
|
||||
public float[] getColor(String type) {
|
||||
|
||||
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.getColor();
|
||||
}
|
||||
return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
|
||||
return defaultColor;
|
||||
}
|
||||
|
||||
|
||||
public boolean isHint(String type, String dossierTemplateId) {
|
||||
public boolean isHint(String type) {
|
||||
|
||||
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isHint();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public boolean isRecommendation(String type) {
|
||||
|
||||
public boolean isRecommendation(String type, String dossierTemplateId) {
|
||||
|
||||
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isRecommendation();
|
||||
}
|
||||
@@ -191,44 +168,8 @@ public class DictionaryService {
|
||||
}
|
||||
|
||||
|
||||
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
|
||||
|
||||
List<DictionaryModel> copy = new ArrayList<>();
|
||||
|
||||
var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
|
||||
dossierTemplateRepresentation.getDictionary().forEach(dm -> {
|
||||
copy.add(SerializationUtils.clone(dm));
|
||||
});
|
||||
|
||||
//TODO merge dictionaries if they have same names
|
||||
long dossierDictionaryVersion = -1;
|
||||
if(dictionariesByDossier.containsKey(dossierId)) {
|
||||
var dossierRepresentation = dictionariesByDossier.get(dossierId);
|
||||
dossierRepresentation.getDictionary().forEach(dm -> {
|
||||
copy.add(SerializationUtils.clone(dm));
|
||||
});
|
||||
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
|
||||
}
|
||||
|
||||
return new Dictionary(copy, DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
|
||||
public Dictionary getThreadSafeDictionary(){
|
||||
return new Dictionary(new ArrayList<>(dictionary), new HashMap<>(localAccessMap), dictionaryVersion);
|
||||
}
|
||||
|
||||
|
||||
public float[] getRequestRemoveColor(String dossierTemplateId) {
|
||||
|
||||
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
|
||||
}
|
||||
|
||||
|
||||
public float[] getNotRedactedColor(String dossierTemplateId) {
|
||||
|
||||
return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
|
||||
}
|
||||
|
||||
|
||||
public float[] getRequestAddColor(String dossierTemplateId) {
|
||||
|
||||
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+31
-87
@@ -1,10 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
@@ -12,13 +11,15 @@ import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -26,23 +27,13 @@ public class DroolsExecutionService {
|
||||
|
||||
private final RulesClient rulesClient;
|
||||
|
||||
private final Map<String, KieContainer> kieContainers = new HashMap<>();
|
||||
@Autowired
|
||||
private KieContainer kieContainer;
|
||||
|
||||
private final Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
|
||||
@Getter
|
||||
private long rulesVersion = -1;
|
||||
|
||||
|
||||
public KieContainer getKieContainer(String dossierTemplateId) {
|
||||
|
||||
KieContainer container = kieContainers.get(dossierTemplateId);
|
||||
if (container == null) {
|
||||
return createOrUpdateKieContainer(dossierTemplateId);
|
||||
} else {
|
||||
return container;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Section executeRules(KieContainer kieContainer, Section section) {
|
||||
public Section executeRules(Section section) {
|
||||
|
||||
KieSession kieSession = kieContainer.newKieSession();
|
||||
kieSession.setGlobal("section", section);
|
||||
@@ -54,81 +45,34 @@ public class DroolsExecutionService {
|
||||
|
||||
}
|
||||
|
||||
public void updateRules() {
|
||||
|
||||
public KieContainer updateRules(String dossierTemplateId) {
|
||||
|
||||
long version = rulesClient.getVersion(dossierTemplateId);
|
||||
Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
|
||||
if (rulesVersion == null) {
|
||||
rulesVersion = -1L;
|
||||
long version = rulesClient.getVersion();
|
||||
if (version > rulesVersion) {
|
||||
rulesVersion = version;
|
||||
updateRules(rulesClient.getRules().getRules());
|
||||
}
|
||||
|
||||
if (version > rulesVersion.longValue()) {
|
||||
rulesVersionPerDossierTemplateId.put(dossierTemplateId, version);
|
||||
return createOrUpdateKieContainer(dossierTemplateId);
|
||||
}
|
||||
return getKieContainer(dossierTemplateId);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieContainer createOrUpdateKieContainer(String dossierTemplateId) {
|
||||
public void updateRules(String drlAsString) {
|
||||
|
||||
try {
|
||||
|
||||
RulesResponse rules = rulesClient.getRules(dossierTemplateId);
|
||||
if (rules == null || StringUtils.isEmpty(rules.getRules())) {
|
||||
if (StringUtils.isEmpty(drlAsString)) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule(dossierTemplateId, rules.getRules(), kieServices);
|
||||
|
||||
var container = kieContainers.get(dossierTemplateId);
|
||||
if (container != null) {
|
||||
container.updateToVersion(kieModule.getReleaseId());
|
||||
return container;
|
||||
}
|
||||
|
||||
container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
kieContainers.put(dossierTemplateId, container);
|
||||
return container;
|
||||
InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
kieContainer.updateToVersion(kieModule.getReleaseId());
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private KieModule getKieModule(String dossierTemplateId, String rules, KieServices kieServices) {
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
InputStream input = new ByteArrayInputStream(rules.getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules" + dossierTemplateId + ".drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
return kieBuilder.getKieModule();
|
||||
}
|
||||
|
||||
|
||||
public void testRules(String rules) {
|
||||
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
KieModule kieModule = getKieModule("test-rules", rules, kieServices);
|
||||
var container = kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
container.newKieSession();
|
||||
container.dispose();
|
||||
}
|
||||
|
||||
|
||||
public long getRulesVersion(String dossierTemplateId) {
|
||||
|
||||
Long rulesVersion = rulesVersionPerDossierTemplateId.get(dossierTemplateId);
|
||||
if (rulesVersion == null) {
|
||||
return -1;
|
||||
}
|
||||
return rulesVersion.longValue();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+170
-291
@@ -1,61 +1,72 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class EntityRedactionService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final SurroundingWordsService surroundingWordsService;
|
||||
|
||||
|
||||
public void processDocument(Document classifiedDoc, String dossierTemplateId, ManualRedactions manualRedactions, String dossierId) {
|
||||
public void processDocument(Document classifiedDoc, ManualRedactions manualRedactions) {
|
||||
|
||||
dictionaryService.updateDictionary(dossierTemplateId, dossierId);
|
||||
KieContainer container = droolsExecutionService.updateRules(dossierTemplateId);
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion(dossierTemplateId);
|
||||
dictionaryService.updateDictionary();
|
||||
droolsExecutionService.updateRules();
|
||||
long rulesVersion = droolsExecutionService.getRulesVersion();
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(dossierTemplateId, dossierId);
|
||||
Dictionary dictionary = dictionaryService.getThreadSafeDictionary();
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, container, manualRedactions, dictionary, false, null));
|
||||
Set<Entity> documentEntities = new HashSet<>(findEntities(classifiedDoc, manualRedactions, dictionary, false, null));
|
||||
|
||||
if (dictionary.hasLocalEntries()) {
|
||||
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
|
||||
documentEntities.stream().forEach(entity -> {
|
||||
if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>())
|
||||
.add(entity);
|
||||
}
|
||||
});
|
||||
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, container, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(documentEntities, foundByLocal, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(documentEntities);
|
||||
Set<Entity> foundByLocal = findEntities(classifiedDoc, manualRedactions, dictionary, true, hintsPerSectionNumber);
|
||||
// HashSet keeps the older value, but we want the new only.
|
||||
documentEntities.removeAll(foundByLocal);
|
||||
documentEntities.addAll(foundByLocal);
|
||||
|
||||
removeEntitiesContainedInLarger(documentEntities);
|
||||
}
|
||||
|
||||
for (Entity entity : documentEntities) {
|
||||
@@ -70,297 +81,136 @@ public class EntityRedactionService {
|
||||
.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
|
||||
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
|
||||
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
|
||||
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
|
||||
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
|
||||
.getLegalBasis()));
|
||||
}
|
||||
}
|
||||
|
||||
dictionaryService.updateExternalDictionary(dictionary, dossierTemplateId);
|
||||
dictionaryService.updateExternalDictionary(dictionary);
|
||||
|
||||
classifiedDoc.setDictionaryVersion(dictionary.getVersion());
|
||||
classifiedDoc.setRulesVersion(rulesVersion);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(Document classifiedDoc, KieContainer kieContainer,
|
||||
ManualRedactions manualRedactions, Dictionary dictionary, boolean local,
|
||||
private Set<Entity> findEntities(Document classifiedDoc, ManualRedactions manualRedactions,
|
||||
Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
Set<Entity> documentEntities = new HashSet<>();
|
||||
|
||||
AtomicInteger sectionNumber = new AtomicInteger(1);
|
||||
int sectionNumber = 1;
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
SearchableText searchableText = paragraph.getSearchableText();
|
||||
|
||||
List<Table> tables = paragraph.getTables();
|
||||
|
||||
for (Table table : tables) {
|
||||
if (table.getColCount() == 2) {
|
||||
sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
} else {
|
||||
sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber));
|
||||
boolean singleCellTable = table.getRowCount() == 1 && table.getColCount() == 1;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
Map<String, CellValue> tabularData = new HashMap<>();
|
||||
int start = 0;
|
||||
for (Cell cell : row) {
|
||||
if (!singleCellTable && cell.isHeaderCell() || CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber);
|
||||
int cellStart = start;
|
||||
cell.getHeaderCells().forEach(headerCell -> {
|
||||
StringBuilder headerBuilder = new StringBuilder();
|
||||
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
|
||||
String headerName = headerBuilder.toString()
|
||||
.replaceAll("\n", "")
|
||||
.replaceAll(" ", "")
|
||||
.replaceAll("-", "");
|
||||
tabularData.put(headerName, new CellValue(cell.getTextBlocks().get(0), cellStart));
|
||||
});
|
||||
start = start + cell.toString().length() + 1; // include automatically appended white space
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
}
|
||||
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber)
|
||||
.tabularData(tabularData)
|
||||
.build(), searchableRow));
|
||||
|
||||
sectionNumber++;
|
||||
}
|
||||
sectionNumber.incrementAndGet();
|
||||
sectionNumber++;
|
||||
}
|
||||
sectionSearchableTextPairs.add(processText(classifiedDoc, paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph
|
||||
.getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, paragraph
|
||||
.getImages()));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
for (Header header : classifiedDoc.getHeaders()) {
|
||||
sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber);
|
||||
Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, dictionary.getDictionaryModels(), local);
|
||||
|
||||
for (Footer footer : classifiedDoc.getFooters()) {
|
||||
sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber).stream())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(paragraph.getHeadline())
|
||||
.sectionNumber(sectionNumber)
|
||||
.build(), searchableText));
|
||||
|
||||
for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) {
|
||||
sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText
|
||||
.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, new ArrayList<>()));
|
||||
sectionNumber.incrementAndGet();
|
||||
sectionNumber++;
|
||||
}
|
||||
|
||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
||||
documentEntities.addAll(analysedRowSection.getEntities());
|
||||
|
||||
for (Image image : analysedRowSection.getImages()) {
|
||||
classifiedDoc.getImages().computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
|
||||
}
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(sectionSearchableTextPair.getSection());
|
||||
documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair.getSearchableText(), dictionary));
|
||||
|
||||
analysedRowSection.getLocalDictionaryAdds().keySet().forEach(key -> {
|
||||
if (dictionary.isRecommendation(key)) {
|
||||
if (dictionary.isRecommendation(key)){
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
if (!dictionary.containsValue(key, value)) {
|
||||
if (!dictionary.containsValue(key, value)){
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key) == null) {
|
||||
log.warn("Dictionary {} is null", key);
|
||||
}
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
|
||||
log.warn("Dictionary {} localEntries is null", key);
|
||||
}
|
||||
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
analysedRowSection.getLocalDictionaryAdds().get(key).forEach( value -> {
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
return documentEntities;
|
||||
}
|
||||
|
||||
|
||||
private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table,
|
||||
ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
SearchableText searchableRow = new SearchableText();
|
||||
Map<String, CellValue> tabularData = new HashMap<>();
|
||||
int start = 0;
|
||||
List<Integer> cellStarts = new ArrayList<>();
|
||||
SectionText sectionText = new SectionText();
|
||||
for (Cell cell : row) {
|
||||
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
|
||||
.get(0)
|
||||
.getSequences()
|
||||
.get(0)
|
||||
.getPage());
|
||||
sectionText.getSectionAreas().add(sectionArea);
|
||||
sectionText.getTextBlocks().addAll(cell.getTextBlocks());
|
||||
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
int cellStart = start;
|
||||
|
||||
if (!cell.isHeaderCell()) {
|
||||
cell.getHeaderCells().forEach(headerCell -> {
|
||||
StringBuilder headerBuilder = new StringBuilder();
|
||||
headerCell.getTextBlocks().forEach(textBlock -> headerBuilder.append(textBlock.getText()));
|
||||
String headerName = headerBuilder.toString()
|
||||
.replaceAll("\n", "")
|
||||
.replaceAll(" ", "")
|
||||
.replaceAll("-", "");
|
||||
sectionArea.setHeader(headerName);
|
||||
tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart));
|
||||
});
|
||||
}
|
||||
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
// TODO avoid cell overlap merging.
|
||||
searchableRow.addAll(textBlock.getSequences());
|
||||
}
|
||||
cellStarts.add(cellStart);
|
||||
start = start + cell.toString().trim().length() + 1;
|
||||
removeEntitiesContainedInLarger(entities);
|
||||
|
||||
for (Entity entity : entities) {
|
||||
if (entity.getPositionSequences().isEmpty()) {
|
||||
entity.setPositionSequences(text.getSequences(entity.getWord(), dictionary.isCaseInsensitiveDictionary(entity
|
||||
.getType()), entity.getTargetSequences()));
|
||||
}
|
||||
Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(searchableRow.getAsStringWithLinebreaks())
|
||||
.searchText(searchableRow.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.tabularData(tabularData)
|
||||
.searchableText(searchableRow)
|
||||
.dictionary(dictionary)
|
||||
.build(), searchableRow));
|
||||
|
||||
if (!local) {
|
||||
sectionText.setText(searchableRow.toString());
|
||||
sectionText.setHeadline(table.getHeadline());
|
||||
sectionText.setSectionNumber(sectionNumber.intValue());
|
||||
sectionText.setTable(true);
|
||||
sectionText.setTabularData(tabularData);
|
||||
sectionText.setCellStarts(cellStarts);
|
||||
classifiedDoc.getSectionText().add(sectionText);
|
||||
}
|
||||
|
||||
sectionNumber.incrementAndGet();
|
||||
}
|
||||
|
||||
return sectionSearchableTextPairs;
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table,
|
||||
ManualRedactions manualRedactions,
|
||||
AtomicInteger sectionNumber, Dictionary dictionary,
|
||||
boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber) {
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
SearchableText entireTableText = new SearchableText();
|
||||
SectionText sectionText = new SectionText();
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (CollectionUtils.isEmpty(cell.getTextBlocks())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!local) {
|
||||
SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight(), cell.getTextBlocks()
|
||||
.get(0)
|
||||
.getSequences()
|
||||
.get(0)
|
||||
.getPage());
|
||||
sectionText.getTextBlocks().addAll(cell.getTextBlocks());
|
||||
sectionText.getSectionAreas().add(sectionArea);
|
||||
}
|
||||
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
entireTableText.addAll(textBlock.getSequences());
|
||||
}
|
||||
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
|
||||
}
|
||||
}
|
||||
|
||||
Set<Entity> rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary);
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : rowEntities)
|
||||
.text(entireTableText.getAsStringWithLinebreaks())
|
||||
.searchText(entireTableText.toString())
|
||||
.headline(table.getHeadline())
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(entireTableText)
|
||||
.dictionary(dictionary)
|
||||
.build(), entireTableText));
|
||||
|
||||
if (!local) {
|
||||
sectionText.setText(entireTableText.toString());
|
||||
sectionText.setHeadline(table.getHeadline());
|
||||
sectionText.setSectionNumber(sectionNumber.intValue());
|
||||
sectionText.setTable(true);
|
||||
classifiedDoc.getSectionText().add(sectionText);
|
||||
}
|
||||
|
||||
return sectionSearchableTextPairs;
|
||||
}
|
||||
|
||||
|
||||
private SectionSearchableTextPair processText(Document classifiedDoc, SearchableText searchableText,
|
||||
List<TextBlock> paragraphTextBlocks, String headline,
|
||||
ManualRedactions manualRedactions, AtomicInteger sectionNumber,
|
||||
Dictionary dictionary, boolean local,
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber,
|
||||
List<PdfImage> images) {
|
||||
|
||||
if (!local) {
|
||||
SectionText sectionText = new SectionText();
|
||||
for (TextBlock paragraphTextBlock : paragraphTextBlocks) {
|
||||
SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock
|
||||
.getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage());
|
||||
sectionText.getSectionAreas().add(sectionArea);
|
||||
}
|
||||
|
||||
sectionText.setText(searchableText.toString());
|
||||
sectionText.setHeadline(headline);
|
||||
sectionText.setSectionNumber(sectionNumber.intValue());
|
||||
sectionText.setTable(false);
|
||||
sectionText.setImages(images.stream()
|
||||
.map(image -> convert(image, sectionNumber.intValue(), headline))
|
||||
.collect(Collectors.toSet()));
|
||||
sectionText.setTextBlocks(paragraphTextBlocks);
|
||||
classifiedDoc.getSectionText().add(sectionText);
|
||||
}
|
||||
|
||||
addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
|
||||
Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
|
||||
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
|
||||
|
||||
return new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(local)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber.intValue()) ? Stream
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber.intValue()).stream())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(searchableText.getAsStringWithLinebreaks())
|
||||
.searchText(searchableText.toString())
|
||||
.headline(headline)
|
||||
.sectionNumber(sectionNumber.intValue())
|
||||
.searchableText(searchableText)
|
||||
.dictionary(dictionary)
|
||||
.images(images.stream()
|
||||
.map(image -> convert(image, sectionNumber.intValue(), headline))
|
||||
.collect(Collectors.toSet()))
|
||||
.build(), searchableText);
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local) {
|
||||
private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
List<DictionaryModel> dictionary, boolean local) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
String searchableString = searchableText.toString();
|
||||
@@ -369,15 +219,58 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
String lowercaseInputString = searchableString.toLowerCase();
|
||||
for (DictionaryModel model : dictionary.getDictionaryModels()) {
|
||||
for (DictionaryModel model : dictionary) {
|
||||
if (model.isCaseInsensitive()) {
|
||||
found.addAll(EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
|
||||
found.addAll(find(lowercaseInputString, model.getValues(local), model.getType(), headline, sectionNumber));
|
||||
} else {
|
||||
found.addAll(EntitySearchUtils.find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber, local, model.isDossierDictionary()));
|
||||
found.addAll(find(searchableString, model.getValues(local), model.getType(), headline, sectionNumber));
|
||||
}
|
||||
}
|
||||
removeEntitiesContainedInLarger(found);
|
||||
return found;
|
||||
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
for (String value : values) {
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(value, stopIndex);
|
||||
stopIndex = startIndex + value.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
|
||||
.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
|
||||
@@ -400,18 +293,4 @@ public class EntityRedactionService {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Image convert(PdfImage pdfImage, int sectionNumber, String headline) {
|
||||
|
||||
return Image.builder()
|
||||
.type(pdfImage.getImageType().equals(ImageType.OTHER) ? "image" : pdfImage.getImageType()
|
||||
.name()
|
||||
.toLowerCase(Locale.ROOT))
|
||||
.position(pdfImage.getPosition())
|
||||
.sectionNumber(sectionNumber)
|
||||
.section(headline)
|
||||
.page(pdfImage.getPage())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.MockMultipartFile;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageClassificationService {
|
||||
|
||||
private final ImageClassificationClient imageClassificationClient;
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
|
||||
public void classifyImages(Page page) {
|
||||
|
||||
page.getImages().forEach(image -> {
|
||||
|
||||
if (settings.isEnableImageClassification() && !isEntirePageImage(image, page)) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ImageIO.write(image.getImage(), "png", baos);
|
||||
var mockFile = new MockMultipartFile("file", "Image.png", "image/png", baos.toByteArray());
|
||||
ImageClassificationResponse response = imageClassificationClient.classify(mockFile);
|
||||
image.setImageType(ImageType.valueOf(response.getCategory()));
|
||||
} catch (Exception e) {
|
||||
log.error("Could not classify image", e);
|
||||
image.setImageType(ImageType.OTHER);
|
||||
}
|
||||
log.info("Image classification took: " + (System.currentTimeMillis() - start));
|
||||
} else {
|
||||
image.setImageType(ImageType.OTHER);
|
||||
}
|
||||
|
||||
image.getImage().flush();
|
||||
image.setImage(null);
|
||||
|
||||
if (image.getImageType().equals(ImageType.OTHER)) {
|
||||
page.getTextBlocks().forEach(textblock -> {
|
||||
if (image.getPosition()
|
||||
.contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
|
||||
image.setImageType(ImageType.OCR);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private boolean isEntirePageImage(PdfImage image, Page page){
|
||||
double imageArea = image.getPosition().getHeight() * image.getPosition().getWidth();
|
||||
if(imageArea / page.getCropBoxArea() >= settings.getMaxImageCropboxRatio()){
|
||||
log.info("Skipping image classification because images is almost as large as the entire page");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
-295
@@ -1,295 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.file.management.v1.api.model.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ReanalyzeService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
private final SurroundingWordsService surroundingWordsService;
|
||||
private final EntityRedactionService entityRedactionService;
|
||||
private final RedactionLogCreatorService redactionLogCreatorService;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final RedactionChangeLogService redactionChangeLogService;
|
||||
private final AnalyzeResponseService analyzeResponseService;
|
||||
private final LegalBasisClient legalBasisClient;
|
||||
|
||||
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
var pageCount = 0;
|
||||
Document classifiedDoc;
|
||||
|
||||
try {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
|
||||
.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
pageCount = classifiedDoc.getPages().size();
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
log.info("Document structure analysis successful, starting redaction analysis...");
|
||||
|
||||
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest
|
||||
.getDossierId());
|
||||
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
|
||||
.getDossierTemplateId());
|
||||
|
||||
log.info("Redaction analysis successful...");
|
||||
|
||||
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(),legalBasis,
|
||||
classifiedDoc.getDictionaryVersion().getDossierTemplateVersion(),
|
||||
classifiedDoc.getDictionaryVersion().getDossierVersion(),
|
||||
classifiedDoc.getRulesVersion(),
|
||||
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
|
||||
log.info("Analyzed with rules {} and dictionary {} for dossierTemplate: {}", classifiedDoc.getRulesVersion(), classifiedDoc
|
||||
.getDictionaryVersion(), analyzeRequest.getDossierTemplateId());
|
||||
|
||||
// first create changelog - this only happens when we migrate files analyzed via the old process and we don't want to loose changeLog data
|
||||
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
// store redactionLog
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc
|
||||
.getSectionText()));
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc
|
||||
.getSectionGrid());
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, pageCount, redactionLog, changeLog);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
|
||||
// not yet ready for reanalysis
|
||||
if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
|
||||
return analyze(analyzeRequest);
|
||||
}
|
||||
|
||||
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
|
||||
.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
|
||||
|
||||
Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
|
||||
Map<String, List<Comment>> comments = null;
|
||||
Set<ManualRedactionEntry> manualAdds = null;
|
||||
|
||||
if (analyzeRequest.getManualRedactions() != null) {
|
||||
// TODO comments will be removed from redactionLog, so we ignore this first.
|
||||
comments = analyzeRequest.getManualRedactions().getComments();
|
||||
manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd();
|
||||
}
|
||||
|
||||
Set<Integer> sectionsToReanalyse = new HashSet<>();
|
||||
Map<Integer, Set<Image>> imageEntries = new HashMap<>();
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
|
||||
sectionsToReanalyse.add(entry.getSectionNumber());
|
||||
}
|
||||
if (entry.isImage() || entry.getType().equals("image")) {
|
||||
imageEntries.computeIfAbsent(entry.getSectionNumber(), x -> new HashSet<>()).add(convert(entry));
|
||||
}
|
||||
}
|
||||
|
||||
for (SectionText sectionText : text.getSectionTexts()) {
|
||||
|
||||
if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrement.getValues())) {
|
||||
sectionsToReanalyse.add(sectionText.getSectionNumber());
|
||||
}
|
||||
|
||||
if (manualAdds != null) {
|
||||
for (SectionArea sectionArea : sectionText.getSectionAreas()) {
|
||||
for (ManualRedactionEntry manualAdd : manualAdds) {
|
||||
for (Rectangle manualPosition : manualAdd.getPositions()) {
|
||||
if (sectionArea.contains(manualPosition)) {
|
||||
manualAdd.setSection(sectionText.getHeadline());
|
||||
manualAdd.setSectionNumber(sectionText.getSectionNumber());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
|
||||
|
||||
if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
|
||||
}
|
||||
|
||||
List<SectionText> reanalysisSections = new ArrayList<>();
|
||||
|
||||
for (SectionText sectionText : text.getSectionTexts()) {
|
||||
|
||||
if (sectionsToReanalyse.contains(sectionText.getSectionNumber())) {
|
||||
reanalysisSections.add(sectionText);
|
||||
}
|
||||
}
|
||||
|
||||
//--
|
||||
|
||||
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
|
||||
|
||||
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
|
||||
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
for (SectionText reanalysisSection : reanalysisSections) {
|
||||
|
||||
Set<Entity> entities = entityRedactionService.findEntities(reanalysisSection.getSearchableText(), reanalysisSection
|
||||
.getHeadline(), reanalysisSection.getSectionNumber(), dictionary, false);
|
||||
if (reanalysisSection.getCellStarts() != null) {
|
||||
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
|
||||
.getCellStarts());
|
||||
} else {
|
||||
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
|
||||
}
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(false)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(entities)
|
||||
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
|
||||
.searchText(reanalysisSection.getSearchableText().toString())
|
||||
.headline(reanalysisSection.getHeadline())
|
||||
.sectionNumber(reanalysisSection.getSectionNumber())
|
||||
.tabularData(reanalysisSection.getTabularData())
|
||||
.searchableText(reanalysisSection.getSearchableText())
|
||||
.dictionary(dictionary)
|
||||
.images(reanalysisSection.getImages())
|
||||
.build(), reanalysisSection.getSearchableText()));
|
||||
}
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
|
||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
||||
Section analysedRowSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
||||
entities.addAll(analysedRowSection.getEntities());
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
|
||||
for (Image image : analysedRowSection.getImages()) {
|
||||
imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
|
||||
for (Entity entity : entities) {
|
||||
Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
|
||||
.add(entityPositionSequence);
|
||||
}
|
||||
|
||||
for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
|
||||
entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
|
||||
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
|
||||
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
|
||||
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
|
||||
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry()));
|
||||
}
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
|
||||
for (int page = 1; page <= text.getNumberOfPages(); page++) {
|
||||
if (entitiesPerPage.get(page) != null) {
|
||||
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
|
||||
.getDossierTemplateId()));
|
||||
}
|
||||
|
||||
if (imagesPerPage.get(page) != null) {
|
||||
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
|
||||
.getDossierTemplateId()));
|
||||
}
|
||||
|
||||
newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
|
||||
.getDossierTemplateId()));
|
||||
}
|
||||
|
||||
redactionLog.getRedactionLogEntry()
|
||||
.removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
|
||||
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
|
||||
|
||||
}
|
||||
|
||||
|
||||
private AnalyzeResult finalizeAnalysis(@RequestBody AnalyzeRequest analyzeRequest, long startTime,
|
||||
RedactionLog redactionLog, Text text,
|
||||
DictionaryIncrement dictionaryIncrement) {
|
||||
|
||||
redactionLog.setDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierTemplateVersion());
|
||||
redactionLog.setDossierDictionaryVersion(dictionaryIncrement.getDictionaryVersion().getDossierVersion());
|
||||
|
||||
var changeLog = redactionChangeLogService.createAndStoreChangeLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog);
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
|
||||
return analyzeResponseService.createAnalyzeResponse(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), duration, text
|
||||
.getNumberOfPages(), redactionLog, changeLog);
|
||||
}
|
||||
|
||||
|
||||
private Set<String> getForceAndRemoveIds(ManualRedactions manualRedactions) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return new HashSet<>();
|
||||
}
|
||||
|
||||
return Stream.concat(manualRedactions.getIdsToRemove()
|
||||
.stream()
|
||||
.map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
public Image convert(RedactionLogEntry entry) {
|
||||
|
||||
Rectangle position = entry.getPositions().get(0);
|
||||
|
||||
return Image.builder()
|
||||
.type(entry.getType())
|
||||
.position(new RedRectangle2D(position.getTopLeft().getX(), position.getTopLeft()
|
||||
.getY(), position.getWidth(), position.getHeight()))
|
||||
.sectionNumber(entry.getSectionNumber())
|
||||
.section(entry.getSection())
|
||||
.page(position.getPage())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.file.management.v1.api.model.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionChangeLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionChangeLogService {
|
||||
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
public RedactionChangeLog createAndStoreChangeLog(String dossierId, String fileId, RedactionLog currentRedactionLog) {
|
||||
|
||||
try {
|
||||
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
|
||||
var changeLog = createChangeLog(currentRedactionLog, previousRedactionLog);
|
||||
redactionStorageService.storeObject(dossierId, fileId, FileType.REDACTION_CHANGELOG, changeLog);
|
||||
return changeLog;
|
||||
} catch (Exception e) {
|
||||
log.debug("Previous redaction log not available");
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private RedactionChangeLog createChangeLog(RedactionLog currentRedactionLog, RedactionLog previousRedactionLog) {
|
||||
|
||||
|
||||
if (previousRedactionLog == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> added = new ArrayList<>(currentRedactionLog.getRedactionLogEntry());
|
||||
added.removeAll(previousRedactionLog.getRedactionLogEntry());
|
||||
|
||||
List<RedactionLogEntry> removed = new ArrayList<>(previousRedactionLog.getRedactionLogEntry());
|
||||
removed.removeAll(currentRedactionLog.getRedactionLogEntry());
|
||||
|
||||
List<RedactionChangeLogEntry> changeLogEntries = added.stream()
|
||||
.map(entry -> convert(entry, ChangeType.ADDED))
|
||||
.collect(Collectors.toList());
|
||||
changeLogEntries.addAll(removed.stream()
|
||||
.map(entry -> convert(entry, ChangeType.REMOVED))
|
||||
.collect(Collectors.toList()));
|
||||
|
||||
return new RedactionChangeLog(changeLogEntries,
|
||||
currentRedactionLog.getDictionaryVersion(),
|
||||
currentRedactionLog.getDossierDictionaryVersion(),
|
||||
currentRedactionLog.getRulesVersion(),
|
||||
currentRedactionLog.getLegalBasisVersion());
|
||||
}
|
||||
|
||||
|
||||
private RedactionChangeLogEntry convert(RedactionLogEntry entry, ChangeType changeType) {
|
||||
|
||||
return RedactionChangeLogEntry.builder()
|
||||
.id(entry.getId())
|
||||
.type(entry.getType())
|
||||
.value(entry.getValue())
|
||||
.reason(entry.getReason())
|
||||
.matchedRule(entry.getMatchedRule())
|
||||
.legalBasis(entry.getLegalBasis())
|
||||
.redacted(entry.isRedacted())
|
||||
.isHint(entry.isHint())
|
||||
.isRecommendation(entry.isRecommendation())
|
||||
.section(entry.getSection())
|
||||
.color(entry.getColor())
|
||||
.positions(entry.getPositions())
|
||||
.sectionNumber(entry.getSectionNumber())
|
||||
.manual(entry.isManual())
|
||||
.status(entry.getStatus())
|
||||
.manualRedactionType(entry.getManualRedactionType())
|
||||
.isDictionaryEntry(entry.isDictionaryEntry())
|
||||
.textBefore(entry.getTextBefore())
|
||||
.textAfter(entry.getTextAfter())
|
||||
.comments(entry.getComments())
|
||||
.changeType(changeType)
|
||||
.isDossierDictionaryEntry(entry.isDossierDictionaryEntry())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
-483
@@ -1,483 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionLogCreatorService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
|
||||
String dossierTemplateId) {
|
||||
|
||||
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
|
||||
|
||||
for (int page = 1; page <= numberOfPages; page++) {
|
||||
|
||||
addSectionGrid(classifiedDoc, page);
|
||||
|
||||
if (classifiedDoc.getEntities().get(page) != null) {
|
||||
classifiedDoc.getRedactionLogEntities()
|
||||
.addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId));
|
||||
}
|
||||
|
||||
if (manualRedactionPages.contains(page)) {
|
||||
classifiedDoc.getRedactionLogEntities()
|
||||
.addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId));
|
||||
}
|
||||
|
||||
if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
|
||||
classifiedDoc.getRedactionLogEntities()
|
||||
.addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
|
||||
int pageNumber, String dossierTemplateId) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
|
||||
for (Image image : images.get(pageNumber)) {
|
||||
|
||||
String id = IdBuilder.buildId(image.getPosition(), pageNumber);
|
||||
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForImage(image, dossierTemplateId, false))
|
||||
.isImage(true)
|
||||
.type(image.getType())
|
||||
.redacted(image.isRedaction())
|
||||
.reason(image.getRedactionReason())
|
||||
.legalBasis(image.getLegalBasis())
|
||||
.matchedRule(image.getMatchedRule())
|
||||
.isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
|
||||
.manual(false)
|
||||
.isDictionaryEntry(false)
|
||||
.isRecommendation(false)
|
||||
.positions(List.of(new Rectangle(new Point((float) image.getPosition()
|
||||
.getX(), (float) image.getPosition().getY()), (float) image.getPosition()
|
||||
.getWidth(), (float) image.getPosition().getHeight(), pageNumber)))
|
||||
.sectionNumber(image.getSectionNumber())
|
||||
.section(image.getSection())
|
||||
.build();
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
|
||||
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
|
||||
if (manualRemoval.getId().equals(id)) {
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
|
||||
image.setRedaction(false);
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
manualOverrideReason = image.getRedactionReason() + ", removed by manual override";
|
||||
redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
|
||||
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = image.getRedactionReason() + ", requested to remove";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
|
||||
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
|
||||
if (manualForceRedact.getId().equals(id)) {
|
||||
String manualOverrideReason = null;
|
||||
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
|
||||
image.setRedaction(true);
|
||||
redactionLogEntry.setRedacted(true);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, false));
|
||||
manualOverrideReason = image.getRedactionReason() + ", forced by manual override";
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = image.getRedactionReason() + ", requested to force redact";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
redactionLogEntry.setColor(getColorForImage(image, dossierTemplateId, true));
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntities.add(redactionLogEntry);
|
||||
}
|
||||
|
||||
return redactionLogEntities;
|
||||
}
|
||||
|
||||
|
||||
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
|
||||
|
||||
Set<Integer> manualRedactionPages = new HashSet<>();
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
manualRedactions.getEntriesToAdd().forEach(entry -> {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
manualRedactionPages.add(pos.getPage());
|
||||
});
|
||||
});
|
||||
return manualRedactionPages;
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
|
||||
int page, String dossierTemplateId) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
|
||||
// Duplicates can exist due table extraction colums over multiple rows.
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
|
||||
entityLoop:
|
||||
for (Entity entity : entities.get(page)) {
|
||||
|
||||
List<Comment> comments = null;
|
||||
|
||||
for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId);
|
||||
if (processedIds.contains(entityPositionSequence.getId())) {
|
||||
|
||||
// TODO refactor this outer loop jump as soon as we have the time.
|
||||
continue entityLoop;
|
||||
} else {
|
||||
processedIds.add(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
|
||||
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
|
||||
if (manualRemoval.getId().equals(entityPositionSequence.getId())) {
|
||||
comments = manualRedactions.getComments().get(manualRemoval.getId());
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
|
||||
entity.setRedaction(false);
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
|
||||
redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
|
||||
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
|
||||
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
|
||||
if (manualForceRedact.getId().equals(entityPositionSequence.getId())) {
|
||||
String manualOverrideReason = null;
|
||||
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
|
||||
entity.setRedaction(true);
|
||||
redactionLogEntry.setRedacted(true);
|
||||
redactionLogEntry.setStatus(Status.APPROVED);
|
||||
redactionLogEntry.setColor(getColor(entity, dossierTemplateId, false));
|
||||
manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
|
||||
manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
|
||||
redactionLogEntry.setStatus(Status.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(entity, dossierTemplateId, true));
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(Status.DECLINED);
|
||||
}
|
||||
|
||||
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
|
||||
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
|
||||
.stream()
|
||||
.flatMap(seq -> seq.getTextPositions().stream())
|
||||
.collect(Collectors.toList()), page);
|
||||
|
||||
if (manualRedactions != null) {
|
||||
comments = manualRedactions.getComments().get(entityPositionSequence.getId());
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(comments);
|
||||
redactionLogEntry.getPositions().addAll(rectanglesPerLine);
|
||||
|
||||
}
|
||||
|
||||
redactionLogEntry.setId(entityPositionSequence.getId());
|
||||
|
||||
// FIXME ids should never be null. Figure out why this happens.
|
||||
if (redactionLogEntry.getId() != null) {
|
||||
redactionLogEntities.add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return redactionLogEntities;
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
|
||||
|
||||
List<Rectangle> rectangles = new ArrayList<>();
|
||||
if (textPositions.size() == 1) {
|
||||
rectangles.add( TextPositionSequence.fromData(textPositions, page).getRectangle());
|
||||
} else {
|
||||
float y = textPositions.get(0).getYDirAdj();
|
||||
int startIndex = 0;
|
||||
for (int i = 1; i < textPositions.size(); i++) {
|
||||
float yDirAdj = textPositions.get(i).getYDirAdj();
|
||||
if (yDirAdj != y) {
|
||||
rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, i), page).getRectangle());
|
||||
y = yDirAdj;
|
||||
startIndex = i;
|
||||
}
|
||||
}
|
||||
if (startIndex != textPositions.size()) {
|
||||
rectangles.add( TextPositionSequence.fromData(textPositions.subList(startIndex, textPositions.size()), page).getRectangle());
|
||||
}
|
||||
}
|
||||
|
||||
return rectangles;
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
|
||||
Map<String, List<Comment>> comments, int page,
|
||||
String dossierTemplateId) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
|
||||
if (manualAdds == null) {
|
||||
return redactionLogEntities;
|
||||
}
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
|
||||
|
||||
String id = manualRedactionEntry.getId();
|
||||
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId);
|
||||
|
||||
List<Rectangle> rectanglesOnPage = new ArrayList<>();
|
||||
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
|
||||
if (page == rectangle.getPage()) {
|
||||
rectanglesOnPage.add(rectangle);
|
||||
redactionLogEntry.getPositions().add(rectangle);
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setComments(comments.get(id));
|
||||
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
|
||||
redactionLogEntities.add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
return redactionLogEntities;
|
||||
}
|
||||
|
||||
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
|
||||
String dossierTemplateId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.sectionNumber(manualRedactionEntry.getSectionNumber())
|
||||
.manual(true)
|
||||
.status(manualRedactionEntry.getStatus())
|
||||
.manualRedactionType(ManualRedactionType.ADD)
|
||||
.isDictionaryEntry(false)
|
||||
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.color(getColor(entity, dossierTemplateId, false))
|
||||
.reason(entity.getRedactionReason())
|
||||
.legalBasis(entity.getLegalBasis())
|
||||
.value(entity.getWord())
|
||||
.type(entity.getType())
|
||||
.redacted(entity.isRedaction())
|
||||
.isHint(isHint(entity, dossierTemplateId))
|
||||
.isRecommendation(isRecommendation(entity, dossierTemplateId))
|
||||
.section(entity.getHeadline())
|
||||
.sectionNumber(entity.getSectionNumber())
|
||||
.matchedRule(entity.getMatchedRule())
|
||||
.isDictionaryEntry(entity.isDictionaryEntry())
|
||||
.textAfter(entity.getTextAfter())
|
||||
.textBefore(entity.getTextBefore())
|
||||
.startOffset(entity.getStart())
|
||||
.endOffset(entity.getEnd())
|
||||
.isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(Entity entity, String dossierTemplateId, boolean requestedToRemove) {
|
||||
|
||||
if (requestedToRemove) {
|
||||
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
|
||||
}
|
||||
if (!entity.isRedaction() && !isHint(entity, dossierTemplateId)) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return dictionaryService.getColor(entity.getType(), dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
|
||||
|
||||
if (status.equals(Status.REQUESTED)) {
|
||||
return dictionaryService.getRequestAddColor(dossierTemplateId);
|
||||
} else if (status.equals(Status.DECLINED)) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return getColor(type, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String dossierTemplateId) {
|
||||
|
||||
return dictionaryService.getColor(type, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private float[] getColorForImage(Image image, String dossierTemplateId, boolean requestedToRemove) {
|
||||
|
||||
if (requestedToRemove) {
|
||||
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
|
||||
}
|
||||
if (!image.isRedaction() && !dictionaryService.isHint(image.getType(), dossierTemplateId)) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return dictionaryService.getColor(image.getType(), dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isHint(Entity entity, String dossierTemplateId) {
|
||||
|
||||
return dictionaryService.isHint(entity.getType(), dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private boolean isRecommendation(Entity entity, String dossierTemplateId) {
|
||||
|
||||
return dictionaryService.isRecommendation(entity.getType(), dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private void addSectionGrid(Document classifiedDoc, int page) {
|
||||
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
|
||||
AbstractTextContainer textBlock = paragraph.getPageBlocks().get(i);
|
||||
|
||||
if (textBlock.getPage() != page) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (textBlock instanceof TextBlock) {
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size()));
|
||||
|
||||
} else if (textBlock instanceof Table) {
|
||||
|
||||
List<CellRectangle> cellRectangles = new ArrayList<>();
|
||||
for (List<Cell> row : ((Table) textBlock).getRows()) {
|
||||
for (Cell cell : row) {
|
||||
if (cell != null) {
|
||||
cellRectangles.add(new CellRectangle(new Point((float) cell.getX(), (float) cell.getY()), (float) cell
|
||||
.getWidth(), (float) cell.getHeight()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
classifiedDoc.getSectionGrid()
|
||||
.getRectanglesPerPage()
|
||||
.computeIfAbsent(page, (x) -> new ArrayList<>())
|
||||
.add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock
|
||||
.getHeight(), i + 1, paragraph.getPageBlocks().size(), cellRectangles));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-140
@@ -1,140 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class SurroundingWordsService {
|
||||
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
findSurroundingWords(entity, searchableText.toString(), entity.getStart(), entity.getEnd());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary,
|
||||
List<Integer> cellstarts) {
|
||||
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String searchableString = searchableText.toString();
|
||||
|
||||
if (cellstarts != null) {
|
||||
for (int i = 0; i < cellstarts.size(); i++) {
|
||||
|
||||
int startOffset = cellstarts.get(i);
|
||||
int endOffset = -1;
|
||||
|
||||
if (i + 1 < cellstarts.size()) {
|
||||
endOffset = cellstarts.get(i + 1);
|
||||
} else {
|
||||
endOffset = searchableString.length() - 1;
|
||||
}
|
||||
|
||||
String text = searchableString.substring(startOffset, endOffset);
|
||||
for (Entity entity : entities) {
|
||||
|
||||
if (dictionary.isHint(entity.getType())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
|
||||
int entityStartOffset = entity.getStart() - startOffset;
|
||||
int entityEndOffset = entity.getEnd() - startOffset;
|
||||
findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not get surrounding text!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
|
||||
|
||||
int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings
|
||||
.getSurroundingWordsOffsetWindow();
|
||||
String textBefore = text.substring(offsetBefore, entityStartOffset);
|
||||
if (!textBefore.isBlank()) {
|
||||
String[] wordsBefore = textBefore.split(" ");
|
||||
int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsBefore.length;
|
||||
if (wordsBefore.length > 0) {
|
||||
entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
|
||||
}
|
||||
}
|
||||
|
||||
int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text
|
||||
.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
|
||||
String textAfter = text.substring(entityEndOffset, endOffset);
|
||||
if (!textAfter.isBlank()) {
|
||||
String[] wordsAfter = textAfter.split(" ");
|
||||
int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings
|
||||
.getNumberOfSurroundingWords() : wordsAfter.length;
|
||||
if (wordsAfter.length > 0) {
|
||||
entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsBefore(String[] words, int number, boolean endWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
int startNumber = words.length > number ? words.length - number : 0;
|
||||
|
||||
for (int i = startNumber; i < words.length; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return endWithSpace ? result + " " : result;
|
||||
}
|
||||
|
||||
|
||||
private String concatWordsAfter(String[] words, int number, boolean startWithSpace) {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < number; i++) {
|
||||
sb.append(words[i]).append(" ");
|
||||
}
|
||||
|
||||
String result = sb.toString().trim();
|
||||
return startWithSpace ? " " + result : result;
|
||||
}
|
||||
|
||||
}
|
||||
-156
@@ -1,156 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@UtilityClass
|
||||
@SuppressWarnings("PMD")
|
||||
public class EntitySearchUtils {
|
||||
|
||||
|
||||
public boolean sectionContainsAny(String sectionText, Set<DictionaryIncrementValue> values) {
|
||||
|
||||
String inputString = sectionText.toLowerCase(Locale.ROOT);
|
||||
|
||||
for (DictionaryIncrementValue value : values) {
|
||||
|
||||
String cleanValue = value.getValue().toLowerCase(Locale.ROOT).trim();
|
||||
|
||||
if (cleanValue.length() <= 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(cleanValue, stopIndex);
|
||||
stopIndex = startIndex + cleanValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
if (value.isCaseinsensitive() || !value.isCaseinsensitive() && sectionText.substring(startIndex, stopIndex).equals(value.getValue())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
|
||||
boolean local, boolean isDossierDictionary) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
for (String value : values) {
|
||||
|
||||
String cleanValue = value.trim();
|
||||
|
||||
if (cleanValue.length() <= 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int startIndex;
|
||||
int stopIndex = 0;
|
||||
do {
|
||||
startIndex = inputString.indexOf(cleanValue, stopIndex);
|
||||
stopIndex = startIndex + cleanValue.length();
|
||||
|
||||
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
|
||||
.charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
|
||||
found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber, !local, isDossierDictionary));
|
||||
}
|
||||
} while (startIndex > -1);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSeparator(char c) {
|
||||
|
||||
return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
|
||||
}
|
||||
|
||||
|
||||
public Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text, Dictionary dictionary) {
|
||||
|
||||
Map<String, List<Entity>> entitiesByWord = new HashMap<>();
|
||||
|
||||
for (Entity entity : entities) {
|
||||
entitiesByWord.computeIfAbsent(entity.getWord(), (x) -> new ArrayList<>()).add(entity);
|
||||
}
|
||||
|
||||
for (String word : entitiesByWord.keySet()) {
|
||||
|
||||
List<Entity> orderedEntities = entitiesByWord.get(word)
|
||||
.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
Entity firstEntity = orderedEntities.get(0);
|
||||
List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(), dictionary.isCaseInsensitiveDictionary(firstEntity
|
||||
.getType()), firstEntity.getTargetSequences());
|
||||
|
||||
for (int i = 0; i <= orderedEntities.size() - 1; i++) {
|
||||
try {
|
||||
orderedEntities.get(i).setPositionSequences(List.of(positionSequences.get(i)));
|
||||
} catch (Exception e) {
|
||||
log.warn("Mismatch between EntityPositionSequence and found Entity!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
removeEntitiesContainedInLarger(entities);
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
|
||||
public void removeEntitiesContainedInLarger(Set<Entity> entities) {
|
||||
|
||||
List<Entity> wordsToRemove = new ArrayList<>();
|
||||
for (Entity word : entities) {
|
||||
for (Entity inner : entities) {
|
||||
if (inner.getWord().length() < word.getWord()
|
||||
.length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word
|
||||
.getSectionNumber() == inner.getSectionNumber()) {
|
||||
wordsToRemove.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
entities.removeAll(wordsToRemove);
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Set<Entity> found, Dictionary dictionary) {
|
||||
found.forEach(f -> addEntitiesWithHigherRank(entities, f, dictionary));
|
||||
}
|
||||
|
||||
|
||||
public void addEntitiesWithHigherRank(Set<Entity> entities, Entity found, Dictionary dictionary) {
|
||||
|
||||
if (entities.contains(found)) {
|
||||
Entity existing = entities.stream().filter(entity -> entity.equals(found)).findFirst().get();
|
||||
if (dictionary.getDictionaryRank(existing.getType()) <= dictionary.getDictionaryRank(found.getType())) {
|
||||
entities.remove(found);
|
||||
}
|
||||
}
|
||||
entities.add(found);
|
||||
}
|
||||
|
||||
public void addEntitiesIgnoreRank(Set<Entity> entities, Set<Entity> found) {
|
||||
// HashSet keeps old value but we want the new.
|
||||
entities.removeAll(found);
|
||||
entities.addAll(found);
|
||||
}
|
||||
}
|
||||
+4
-11
@@ -1,13 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.hash.HashFunction;
|
||||
import com.google.common.hash.Hashing;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class IdBuilder {
|
||||
@@ -23,11 +23,4 @@ public class IdBuilder {
|
||||
|
||||
return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
public String buildId(RedRectangle2D rectangle2D, int page) {
|
||||
return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page, StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-18
@@ -1,28 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class Patterns {
|
||||
|
||||
public static Map<String, Pattern> patternCache = new HashMap<>();
|
||||
|
||||
public static Pattern AUTHOR_TABLE_SPITTER = Pattern.compile("((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2}\\.)+|((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,}( ?[A-ZÄÖÜ]{1,2} )+");
|
||||
|
||||
|
||||
public Pattern getCompiledPattern(String pattern, boolean caseInsensitive) {
|
||||
|
||||
String patternKey = pattern + caseInsensitive;
|
||||
if (patternCache.containsKey(patternKey)) {
|
||||
return patternCache.get(patternKey);
|
||||
}
|
||||
Pattern compiledPattern = Pattern.compile(pattern, caseInsensitive ? Pattern.CASE_INSENSITIVE : 0);
|
||||
patternCache.put(patternKey, compiledPattern);
|
||||
return compiledPattern;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -1,7 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
@@ -10,6 +8,8 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class ResourceLoader {
|
||||
|
||||
@@ -27,4 +27,4 @@ public class ResourceLoader {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+2
-3
@@ -7,12 +7,11 @@ public class TextNormalizationUtilities {
|
||||
|
||||
/**
|
||||
* Revert hyphenation due to line breaks.
|
||||
*
|
||||
* @param text Text to be processed.
|
||||
* @return Text without line-break hyphenation.
|
||||
*/
|
||||
public static String removeHyphenLineBreaks(String text) {
|
||||
return text.replaceAll("([^\\s\\d\\-]{2,})[\\-\\u00AD]\\R|\n\r(.+ )", "$1$2");
|
||||
return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+59
-254
@@ -1,292 +1,98 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Graphics;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.ParsedElements;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class PdfSegmentationService {
|
||||
|
||||
private final static int MAX_PAGES_BEFORE_GC = 250;
|
||||
|
||||
private final RulingCleaningService rulingCleaningService;
|
||||
private final TableExtractionService tableExtractionService;
|
||||
private final BlockificationService blockificationService;
|
||||
private final ClassificationService classificationService;
|
||||
private final SectionsBuilderService sectionsBuilderService;
|
||||
private final ImageClassificationService imageClassificationService;
|
||||
|
||||
|
||||
public Document parseDocument(InputStream documentInputStream) throws IOException {
|
||||
return parseDocument(documentInputStream, false);
|
||||
}
|
||||
public Document parseDocument(PDDocument pdDocument) throws IOException {
|
||||
|
||||
public Document parseDocument(InputStream documentInputStream, boolean ignoreImages) throws IOException {
|
||||
PDDocument pdDocument = null;
|
||||
try {
|
||||
//create tempFile
|
||||
File tempFile = File.createTempFile("document", ".pdf");
|
||||
IOUtils.copy(documentInputStream, new FileOutputStream(tempFile));
|
||||
Document document = new Document();
|
||||
|
||||
// initialize required variables
|
||||
Document document = new Document();
|
||||
List<Page> pages = new ArrayList<>();
|
||||
List<Page> pages = new ArrayList<>();
|
||||
PDFLinesTextStripper stripper = new PDFLinesTextStripper();
|
||||
for (int pageNumber = 1; pageNumber <= pdDocument.getNumberOfPages(); pageNumber++) {
|
||||
PDPage pdPage = pdDocument.getPage(pageNumber - 1);
|
||||
stripper.setPageNumber(pageNumber);
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
stripper.getText(pdDocument);
|
||||
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight();
|
||||
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isRotated = rotation != 0 && rotation != 360;
|
||||
|
||||
|
||||
pdDocument = reinitializePDDocument(tempFile, null);
|
||||
long pageCount = pdDocument.getNumberOfPages();
|
||||
ParsedElements parsedElements = ParsedElements.builder()
|
||||
.rulings(stripper.getRulings())
|
||||
.sequences(stripper.getTextPositionSequences())
|
||||
.maxCharWidth(stripper.getMaxCharWidths())
|
||||
.maxCharHeight(stripper.getMaxCharWidths())
|
||||
.landscape(isLandscape)
|
||||
.rotated(isRotated)
|
||||
.build();
|
||||
|
||||
for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
|
||||
.getMaxCharWidth(), parsedElements.getMaxCharHeight());
|
||||
|
||||
if (pageNumber % MAX_PAGES_BEFORE_GC == 0) {
|
||||
pdDocument = reinitializePDDocument(tempFile, pdDocument);
|
||||
}
|
||||
Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
|
||||
.getVertical());
|
||||
page.setRotation(rotation);
|
||||
|
||||
PDFLinesTextStripper stripper = new PDFLinesTextStripper();
|
||||
PDPage pdPage = pdDocument.getPage(pageNumber - 1);
|
||||
stripper.setPageNumber(pageNumber);
|
||||
stripper.setStartPage(pageNumber);
|
||||
stripper.setEndPage(pageNumber);
|
||||
stripper.setPdpage(pdPage);
|
||||
stripper.getText(pdDocument);
|
||||
tableExtractionService.extractTables(cleanRulings, page);
|
||||
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight();
|
||||
buildPageStatistics(page);
|
||||
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isRotated = rotation != 0 && rotation != 360;
|
||||
page.setLandscape(parsedElements.isLandscape() || parsedElements.isRotated());
|
||||
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
|
||||
.getMaxCharHeight());
|
||||
|
||||
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
|
||||
.getVertical());
|
||||
|
||||
PDRectangle cropbox = pdPage.getCropBox();
|
||||
float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
|
||||
page.setCropBoxArea(cropboxArea);
|
||||
|
||||
page.setRotation(rotation);
|
||||
page.setLandscape(isLandscape || isRotated);
|
||||
page.setPageNumber(pageNumber);
|
||||
List<PdfImage> mergedList = processImages(stripper.getImages());
|
||||
|
||||
List<PdfImage> imagesInImage = new ArrayList<>();
|
||||
for(PdfImage image: mergedList){
|
||||
for (PdfImage inner: mergedList){
|
||||
if(image != inner && image.getPosition().contains(inner.getPosition().getX(), inner.getPosition().getY(), inner.getPosition().getWidth(), inner.getPosition().getHeight())){
|
||||
imagesInImage.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
mergedList.removeAll(imagesInImage);
|
||||
|
||||
page.setImages(mergedList);
|
||||
|
||||
tableExtractionService.extractTables(cleanRulings, page);
|
||||
buildPageStatistics(page);
|
||||
increaseDocumentStatistics(page, document);
|
||||
|
||||
|
||||
if (!ignoreImages) {
|
||||
imageClassificationService.classifyImages(page);
|
||||
}
|
||||
|
||||
pages.add(page);
|
||||
|
||||
|
||||
}
|
||||
|
||||
document.setPages(pages);
|
||||
|
||||
classificationService.classifyDocument(document);
|
||||
sectionsBuilderService.buildSections(document);
|
||||
sectionsBuilderService.addImagesToSections(document);
|
||||
|
||||
pdDocument = reinitializePDDocument(tempFile, pdDocument);
|
||||
|
||||
IOUtils.close(pdDocument);
|
||||
|
||||
tempFile.delete();
|
||||
|
||||
return document;
|
||||
} finally {
|
||||
if (pdDocument != null) {
|
||||
pdDocument.close();
|
||||
}
|
||||
page.setPageNumber(pageNumber);
|
||||
increaseDocumentStatistics(page, document);
|
||||
pages.add(page);
|
||||
}
|
||||
}
|
||||
document.setPages(pages);
|
||||
|
||||
private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException {
|
||||
if (pdDocument != null) {
|
||||
pdDocument.close();
|
||||
}
|
||||
System.runFinalization();
|
||||
System.gc();
|
||||
classificationService.classifyDocument(document);
|
||||
|
||||
MemoryStats.printMemoryStats();
|
||||
|
||||
var newPDDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupTempFileOnly());
|
||||
newPDDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
return newPDDocument;
|
||||
}
|
||||
|
||||
//merge images, if they are separated during pdf import, return new list of Pdfimages
|
||||
private List<PdfImage> processImages(List<PdfImage> imageList) {
|
||||
if (imageList.size() > 1) {
|
||||
List<PdfImage> mergedList = new ArrayList<>();
|
||||
int countElementsInList = 0;
|
||||
boolean beginImage = true;
|
||||
|
||||
// a List of Boolean, true = candidate for merging, false = no merging
|
||||
List<Boolean> candidatesList = getCandidatesList(imageList);
|
||||
|
||||
// loop through list, if there are candidates for merging (true), merge images and add it to mergedList
|
||||
for (int i = 0; i < candidatesList.size(); i++) {
|
||||
if (candidatesList.get(i)) {
|
||||
if (beginImage) {
|
||||
//begin of image, merge two parts of imageList
|
||||
PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1));
|
||||
// image merge successful
|
||||
if (mergedImage != null) {
|
||||
mergedList.add(mergedImage);
|
||||
countElementsInList++;
|
||||
}
|
||||
} else {
|
||||
//middle of an image, merge current piece auf mergedList with image of imageList
|
||||
PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1));
|
||||
// image merge successful
|
||||
if (mergedImage != null) {
|
||||
mergedList.set(countElementsInList - 1, mergedImage);
|
||||
}
|
||||
}
|
||||
beginImage = false;
|
||||
} else {
|
||||
// if the last candidate is false, then both images i and i+1 must be added
|
||||
if (i == candidatesList.size() - 1) {
|
||||
if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) {
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
} else {
|
||||
mergedList.add(imageList.get(i));
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
}
|
||||
} else {
|
||||
//first image is not splitted, add i to resultlist
|
||||
if (beginImage) {
|
||||
mergedList.add(imageList.get(i));
|
||||
countElementsInList++;
|
||||
} else {
|
||||
// i is the end of an image, add begin of new image
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
countElementsInList++;
|
||||
beginImage = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return mergedList;
|
||||
} else {
|
||||
return imageList;
|
||||
}
|
||||
}
|
||||
|
||||
private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2) {
|
||||
|
||||
// diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten
|
||||
double width = image1.getPosition().getWidth();
|
||||
double height1 = image1.getPosition().getHeight();
|
||||
double height2 = image2.getPosition().getHeight();
|
||||
// mit den Werten, die unter Image gespeichert sind, funktioniert es
|
||||
double img1height = image1.getImage().getHeight();
|
||||
double img1width = image1.getImage().getWidth();
|
||||
double img2height = image2.getImage().getHeight();
|
||||
|
||||
BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB);
|
||||
Graphics mergedImageGraphics = mergedImage.getGraphics();
|
||||
try {
|
||||
mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null);
|
||||
mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null);
|
||||
|
||||
// set Image, Position and type for merged Image
|
||||
//set position for merged image with values of image1 and the height of both
|
||||
Rectangle2D pos = new Rectangle2D.Float();
|
||||
pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), width, height1 + height2);
|
||||
PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage());
|
||||
// Graphics need to be disposed
|
||||
|
||||
image1.getImage().flush();
|
||||
image2.getImage().flush();
|
||||
|
||||
mergedImage.flush();
|
||||
mergedImageGraphics.dispose();
|
||||
|
||||
return newPdfImage;
|
||||
} catch (Exception e) {
|
||||
// failed to merge image
|
||||
log.error("Failed to merge image", e);
|
||||
return null;
|
||||
}
|
||||
sectionsBuilderService.buildSections(document);
|
||||
|
||||
return document;
|
||||
|
||||
}
|
||||
|
||||
//make a list of true and false, if the image is a candidate for merging
|
||||
private List<Boolean> getCandidatesList(List<PdfImage> imageList) {
|
||||
List<Boolean> candidatesList = new ArrayList<>();
|
||||
for (int i = 0; i < imageList.size(); i++) {
|
||||
if (i >= 1) {
|
||||
candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i)));
|
||||
}
|
||||
}
|
||||
return candidatesList;
|
||||
}
|
||||
|
||||
// evaluate if two images are candidates for merging, depending on their coordinates, width and height
|
||||
private boolean isCandidateForMerging(PdfImage image1, PdfImage image2) {
|
||||
double x1 = image1.getPosition().getX();
|
||||
double y1 = image1.getPosition().getY();
|
||||
double width1 = image1.getPosition().getWidth();
|
||||
double x2 = image2.getPosition().getX();
|
||||
double y2 = image2.getPosition().getY();
|
||||
double width2 = image2.getPosition().getWidth();
|
||||
double height2 = image2.getPosition().getHeight();
|
||||
//if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates,
|
||||
// then it is the same picture and has to be merged -> return true
|
||||
return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(y1 - y2) && width2 > (height2 / 6);
|
||||
}
|
||||
|
||||
|
||||
private void increaseDocumentStatistics(Page page, Document document) {
|
||||
@@ -319,5 +125,4 @@ public class PdfSegmentationService {
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+21
-113
@@ -1,15 +1,21 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
@Service
|
||||
public class SectionsBuilderService {
|
||||
@@ -18,46 +24,25 @@ public class SectionsBuilderService {
|
||||
|
||||
List<AbstractTextContainer> chunkWords = new ArrayList<>();
|
||||
List<Paragraph> chunkBlockList = new ArrayList<>();
|
||||
List<Header> headers = new ArrayList<>();
|
||||
List<Footer> footers = new ArrayList<>();
|
||||
List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
|
||||
String lastHeadline = "";
|
||||
Table previousTable = null;
|
||||
for (Page page : document.getPages()) {
|
||||
List<TextBlock> header = new ArrayList<>();
|
||||
List<TextBlock> footer = new ArrayList<>();
|
||||
List<TextBlock> unclassifiedText = new ArrayList<>();
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
|
||||
if (current.getClassification() == null) {
|
||||
if (current.getClassification() == null || current.getClassification()
|
||||
.equals("Header") || current.getClassification().equals("Footer")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
current.setPage(page.getPageNumber());
|
||||
|
||||
if (current.getClassification().equals("Header")) {
|
||||
header.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.getClassification().equals("Footer")) {
|
||||
footer.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.getClassification().equals("Other")) {
|
||||
unclassifiedText.add((TextBlock) current);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification()
|
||||
.startsWith("H ") || !document.isHeadlines()) {
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if (document.isHeadlines()) {
|
||||
if(document.isHeadlines()) {
|
||||
lastHeadline = current.getText();
|
||||
}
|
||||
chunkBlockList.add(chunkBlock);
|
||||
@@ -75,16 +60,6 @@ public class SectionsBuilderService {
|
||||
chunkWords.add(current);
|
||||
prev = current;
|
||||
}
|
||||
|
||||
if (!header.isEmpty()) {
|
||||
headers.add(new Header(header));
|
||||
}
|
||||
if (!footer.isEmpty()) {
|
||||
footers.add(new Footer(footer));
|
||||
}
|
||||
if (!unclassifiedText.isEmpty()) {
|
||||
unclassifiedTexts.add(new UnclassifiedText(unclassifiedText));
|
||||
}
|
||||
}
|
||||
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
@@ -92,70 +67,6 @@ public class SectionsBuilderService {
|
||||
chunkBlockList.add(chunkBlock);
|
||||
|
||||
document.setParagraphs(chunkBlockList);
|
||||
document.setHeaders(headers);
|
||||
document.setFooters(footers);
|
||||
document.setUnclassifiedTexts(unclassifiedTexts);
|
||||
}
|
||||
|
||||
|
||||
public void addImagesToSections(Document document) {
|
||||
|
||||
Map<Integer, SortedSet<Paragraph>> paragraphMap = new HashMap<>();
|
||||
for (Paragraph paragraph : document.getParagraphs()) {
|
||||
for (AbstractTextContainer container : paragraph.getPageBlocks()) {
|
||||
paragraphMap.computeIfAbsent(container.getPage(), x -> new TreeSet<>()).add(paragraph);
|
||||
}
|
||||
}
|
||||
|
||||
if (paragraphMap.isEmpty()) {
|
||||
Paragraph paragraph = new Paragraph();
|
||||
document.getParagraphs().add(paragraph);
|
||||
paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
|
||||
}
|
||||
|
||||
// first page is always a paragraph, else we can't process pages 1..N,
|
||||
// where N is the first found page with a paragraph
|
||||
if (paragraphMap.get(1) == null) {
|
||||
Paragraph paragraph = new Paragraph();
|
||||
document.getParagraphs().add(paragraph);
|
||||
paragraphMap.computeIfAbsent(1, x -> new TreeSet<>()).add(paragraph);
|
||||
}
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
for (PdfImage image : page.getImages()) {
|
||||
SortedSet<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
|
||||
if (paragraphsOnPage == null) {
|
||||
int i = page.getPageNumber();
|
||||
while (paragraphsOnPage == null) {
|
||||
paragraphsOnPage = paragraphMap.get(i);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
|
||||
Float perviousEnd = 0f;
|
||||
for (Paragraph paragraph : paragraphsOnPage) {
|
||||
Float currentEnd = 0f;
|
||||
for (AbstractTextContainer abs : paragraph.getPageBlocks()) {
|
||||
if (abs.getPage() != page.getPageNumber()) {
|
||||
continue;
|
||||
}
|
||||
if (abs.getMaxY() > currentEnd) {
|
||||
currentEnd = abs.getMaxY();
|
||||
}
|
||||
}
|
||||
|
||||
if (image.getPosition().getY() >= perviousEnd && image.getPosition().getY() <= currentEnd) {
|
||||
paragraph.getImages().add(image);
|
||||
image.setAppendedToParagraph(true);
|
||||
}
|
||||
perviousEnd = currentEnd;
|
||||
}
|
||||
if (!image.isAppendedToParagraph()) {
|
||||
paragraphsOnPage.first().getImages().add(image);
|
||||
image.setAppendedToParagraph(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -264,9 +175,9 @@ public class SectionsBuilderService {
|
||||
|
||||
private boolean hasInvalidHeaderInformation(Table table) {
|
||||
|
||||
return table.getRows()
|
||||
.stream()
|
||||
.flatMap(row -> row.stream().filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
return table.getRows().stream()
|
||||
.flatMap(row -> row.stream()
|
||||
.filter(cell -> CollectionUtils.isNotEmpty(cell.getHeaderCells())))
|
||||
.findAny()
|
||||
.isEmpty();
|
||||
|
||||
@@ -277,9 +188,6 @@ public class SectionsBuilderService {
|
||||
|
||||
for (int i = table.getRowCount() - 1; i >= 0; i--) { // Non header rows are most likely at bottom of table
|
||||
List<Cell> row = table.getRows().get(i);
|
||||
if (row.size() == 1) {
|
||||
continue;
|
||||
}
|
||||
boolean allNonHeader = true;
|
||||
for (Cell cell : row) {
|
||||
if (cell.isHeaderCell()) {
|
||||
@@ -296,4 +204,4 @@ public class SectionsBuilderService {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+8
-8
@@ -1,18 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server.settings;
|
||||
|
||||
import lombok.Data;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@ConfigurationProperties("redaction-service")
|
||||
public class RedactionServiceSettings {
|
||||
|
||||
private int numberOfSurroundingWords = 3;
|
||||
/**
|
||||
* Tenant used in single tenant mode.
|
||||
*/
|
||||
private String defaultTenant = "iqser-id";
|
||||
|
||||
private int surroundingWordsOffsetWindow = 100;
|
||||
private int flattenImageDpi = 100;
|
||||
|
||||
private boolean enableImageClassification = true;
|
||||
|
||||
private float maxImageCropboxRatio = 0.9f;
|
||||
|
||||
}
|
||||
}
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.storage;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.file.management.v1.api.model.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionStorageService {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final StorageService storageService;
|
||||
|
||||
@SneakyThrows
|
||||
public InputStream getStoredObject(String storageId) {
|
||||
return storageService.getObject(storageId).getInputStream();
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any));
|
||||
}
|
||||
|
||||
|
||||
public RedactionLog getRedactionLog(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert RedactionLog", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Text getText(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(inputStreamResource.getInputStream(), Text.class);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert Text", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public SectionGrid getSectionGrid(String dossierId, String fileId) {
|
||||
|
||||
var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
|
||||
try {
|
||||
return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert RedactionLog", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@RequiredArgsConstructor
|
||||
public enum StorageType {
|
||||
PARSED_DOCUMENT(".json");
|
||||
|
||||
@Getter
|
||||
private final String extension;
|
||||
|
||||
}
|
||||
|
||||
public static class StorageIdUtils {
|
||||
|
||||
public static String getStorageId(String dossierId, String fileId, FileType fileType) {
|
||||
return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+3
-9
@@ -1,8 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
@@ -20,8 +18,6 @@ public abstract class AbstractTextContainer {
|
||||
protected String classification;
|
||||
protected int page;
|
||||
|
||||
private Orientation orientation = Orientation.NONE;
|
||||
|
||||
public abstract String getText();
|
||||
|
||||
public boolean contains(AbstractTextContainer other) {
|
||||
@@ -29,17 +25,15 @@ public abstract class AbstractTextContainer {
|
||||
}
|
||||
|
||||
public boolean contains(Rectangle other) {
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public float getHeight() {
|
||||
return maxY - minY;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
||||
public float getWidth() {
|
||||
return maxX - minX;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+8
-17
@@ -1,21 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
@NoArgsConstructor
|
||||
public class Cell extends Rectangle {
|
||||
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
@@ -43,13 +41,8 @@ public class Cell extends Rectangle {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
TextPositionSequence previous = null;
|
||||
while (itty.hasNext()) {
|
||||
|
||||
TextBlock textBlock = itty.next();
|
||||
|
||||
for (TextBlock textBlock : textBlocks) {
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
@@ -61,7 +54,6 @@ public class Cell extends Rectangle {
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
@@ -69,5 +61,4 @@ public class Cell extends Rectangle {
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class CleanRulings {
|
||||
|
||||
+137
-138
@@ -8,171 +8,170 @@ import java.util.List;
|
||||
@SuppressWarnings("all")
|
||||
public class Rectangle extends Rectangle2D.Float {
|
||||
|
||||
protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
|
||||
/**
|
||||
* Ill-defined comparator, from when Rectangle was Comparable.
|
||||
* <p>
|
||||
* see https://github.com/tabulapdf/tabula-java/issues/116
|
||||
*
|
||||
* @deprecated with no replacement
|
||||
*/
|
||||
@Deprecated
|
||||
public static final Comparator<Rectangle> ILL_DEFINED_ORDER = new Comparator<Rectangle>() {
|
||||
@Override
|
||||
public int compare(Rectangle o1, Rectangle o2) {
|
||||
if (o1.equals(o2)) return 0;
|
||||
if (o1.verticalOverlap(o2) > VERTICAL_COMPARISON_THRESHOLD) {
|
||||
return o1.isLtrDominant() == -1 && o2.isLtrDominant() == -1
|
||||
? -java.lang.Double.compare(o1.getX(), o2.getX())
|
||||
: java.lang.Double.compare(o1.getX(), o2.getX());
|
||||
} else {
|
||||
return java.lang.Float.compare(o1.getBottom(), o2.getBottom());
|
||||
}
|
||||
}
|
||||
};
|
||||
/**
|
||||
* Ill-defined comparator, from when Rectangle was Comparable.
|
||||
*
|
||||
* see https://github.com/tabulapdf/tabula-java/issues/116
|
||||
* @deprecated with no replacement
|
||||
*/
|
||||
@Deprecated
|
||||
public static final Comparator<Rectangle> ILL_DEFINED_ORDER = new Comparator<Rectangle>() {
|
||||
@Override public int compare(Rectangle o1, Rectangle o2) {
|
||||
if (o1.equals(o2)) return 0;
|
||||
if (o1.verticalOverlap(o2) > VERTICAL_COMPARISON_THRESHOLD) {
|
||||
return o1.isLtrDominant() == -1 && o2.isLtrDominant() == -1
|
||||
? - java.lang.Double.compare(o1.getX(), o2.getX())
|
||||
: java.lang.Double.compare(o1.getX(), o2.getX());
|
||||
} else {
|
||||
return java.lang.Float.compare(o1.getBottom(), o2.getBottom());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
protected static final float VERTICAL_COMPARISON_THRESHOLD = 0.4f;
|
||||
|
||||
public Rectangle() {
|
||||
super();
|
||||
}
|
||||
public Rectangle() {
|
||||
super();
|
||||
}
|
||||
|
||||
public Rectangle(float top, float left, float width, float height) {
|
||||
super();
|
||||
this.setRect(left, top, width, height);
|
||||
}
|
||||
public Rectangle(float top, float left, float width, float height) {
|
||||
super();
|
||||
this.setRect(left, top, width, height);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param rectangles
|
||||
* @return minimum bounding box that contains all the rectangles
|
||||
*/
|
||||
public static Rectangle boundingBoxOf(List<? extends Rectangle> rectangles) {
|
||||
float minx = java.lang.Float.MAX_VALUE;
|
||||
float miny = java.lang.Float.MAX_VALUE;
|
||||
float maxx = java.lang.Float.MIN_VALUE;
|
||||
float maxy = java.lang.Float.MIN_VALUE;
|
||||
public int compareTo(Rectangle other) {
|
||||
return ILL_DEFINED_ORDER.compare(this, other);
|
||||
}
|
||||
|
||||
for (Rectangle r : rectangles) {
|
||||
minx = (float) Math.min(r.getMinX(), minx);
|
||||
miny = (float) Math.min(r.getMinY(), miny);
|
||||
maxx = (float) Math.max(r.getMaxX(), maxx);
|
||||
maxy = (float) Math.max(r.getMaxY(), maxy);
|
||||
}
|
||||
return new Rectangle(miny, minx, maxx - minx, maxy - miny);
|
||||
}
|
||||
// I'm bad at Java and need this for fancy sorting in
|
||||
// technology.tabula.TextChunk.
|
||||
public int isLtrDominant() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
public int compareTo(Rectangle other) {
|
||||
return ILL_DEFINED_ORDER.compare(this, other);
|
||||
}
|
||||
public float getArea() {
|
||||
return this.width * this.height;
|
||||
}
|
||||
|
||||
// I'm bad at Java and need this for fancy sorting in
|
||||
// technology.tabula.TextChunk.
|
||||
public int isLtrDominant() {
|
||||
return 0;
|
||||
}
|
||||
public float verticalOverlap(Rectangle other) {
|
||||
return Math.max(0, Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
|
||||
}
|
||||
|
||||
public float getArea() {
|
||||
return this.width * this.height;
|
||||
}
|
||||
public boolean verticallyOverlaps(Rectangle other) {
|
||||
return verticalOverlap(other) > 0;
|
||||
}
|
||||
|
||||
public float verticalOverlap(Rectangle other) {
|
||||
return Math.max(0, Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
|
||||
}
|
||||
public float horizontalOverlap(Rectangle other) {
|
||||
return Math.max(0, Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
|
||||
}
|
||||
|
||||
public boolean verticallyOverlaps(Rectangle other) {
|
||||
return verticalOverlap(other) > 0;
|
||||
}
|
||||
public boolean horizontallyOverlaps(Rectangle other) {
|
||||
return horizontalOverlap(other) > 0;
|
||||
}
|
||||
|
||||
public float horizontalOverlap(Rectangle other) {
|
||||
return Math.max(0, Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
|
||||
}
|
||||
public float verticalOverlapRatio(Rectangle other) {
|
||||
float rv = 0, delta = Math.min(this.getBottom() - this.getTop(), other.getBottom() - other.getTop());
|
||||
|
||||
public boolean horizontallyOverlaps(Rectangle other) {
|
||||
return horizontalOverlap(other) > 0;
|
||||
}
|
||||
if (other.getTop() <= this.getTop() && this.getTop() <= other.getBottom()
|
||||
&& other.getBottom() <= this.getBottom()) {
|
||||
rv = (other.getBottom() - this.getTop()) / delta;
|
||||
} else if (this.getTop() <= other.getTop() && other.getTop() <= this.getBottom()
|
||||
&& this.getBottom() <= other.getBottom()) {
|
||||
rv = (this.getBottom() - other.getTop()) / delta;
|
||||
} else if (this.getTop() <= other.getTop() && other.getTop() <= other.getBottom()
|
||||
&& other.getBottom() <= this.getBottom()) {
|
||||
rv = (other.getBottom() - other.getTop()) / delta;
|
||||
} else if (other.getTop() <= this.getTop() && this.getTop() <= this.getBottom()
|
||||
&& this.getBottom() <= other.getBottom()) {
|
||||
rv = (this.getBottom() - this.getTop()) / delta;
|
||||
}
|
||||
|
||||
public float verticalOverlapRatio(Rectangle other) {
|
||||
float rv = 0, delta = Math.min(this.getBottom() - this.getTop(), other.getBottom() - other.getTop());
|
||||
return rv;
|
||||
|
||||
if (other.getTop() <= this.getTop() && this.getTop() <= other.getBottom()
|
||||
&& other.getBottom() <= this.getBottom()) {
|
||||
rv = (other.getBottom() - this.getTop()) / delta;
|
||||
} else if (this.getTop() <= other.getTop() && other.getTop() <= this.getBottom()
|
||||
&& this.getBottom() <= other.getBottom()) {
|
||||
rv = (this.getBottom() - other.getTop()) / delta;
|
||||
} else if (this.getTop() <= other.getTop() && other.getTop() <= other.getBottom()
|
||||
&& other.getBottom() <= this.getBottom()) {
|
||||
rv = (other.getBottom() - other.getTop()) / delta;
|
||||
} else if (other.getTop() <= this.getTop() && this.getTop() <= this.getBottom()
|
||||
&& this.getBottom() <= other.getBottom()) {
|
||||
rv = (this.getBottom() - this.getTop()) / delta;
|
||||
}
|
||||
}
|
||||
|
||||
return rv;
|
||||
public float overlapRatio(Rectangle other) {
|
||||
double intersectionWidth = Math.max(0,
|
||||
Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
|
||||
double intersectionHeight = Math.max(0,
|
||||
Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
|
||||
double intersectionArea = Math.max(0, intersectionWidth * intersectionHeight);
|
||||
double unionArea = this.getArea() + other.getArea() - intersectionArea;
|
||||
|
||||
}
|
||||
return (float) (intersectionArea / unionArea);
|
||||
}
|
||||
|
||||
public float overlapRatio(Rectangle other) {
|
||||
double intersectionWidth = Math.max(0,
|
||||
Math.min(this.getRight(), other.getRight()) - Math.max(this.getLeft(), other.getLeft()));
|
||||
double intersectionHeight = Math.max(0,
|
||||
Math.min(this.getBottom(), other.getBottom()) - Math.max(this.getTop(), other.getTop()));
|
||||
double intersectionArea = Math.max(0, intersectionWidth * intersectionHeight);
|
||||
double unionArea = this.getArea() + other.getArea() - intersectionArea;
|
||||
public Rectangle merge(Rectangle other) {
|
||||
this.setRect(this.createUnion(other));
|
||||
return this;
|
||||
}
|
||||
|
||||
return (float) (intersectionArea / unionArea);
|
||||
}
|
||||
public float getTop() {
|
||||
return (float) this.getMinY();
|
||||
}
|
||||
|
||||
public Rectangle merge(Rectangle other) {
|
||||
this.setRect(this.createUnion(other));
|
||||
return this;
|
||||
}
|
||||
public void setTop(float top) {
|
||||
float deltaHeight = top - this.y;
|
||||
this.setRect(this.x, top, this.width, this.height - deltaHeight);
|
||||
}
|
||||
|
||||
public float getTop() {
|
||||
return (float) this.getMinY();
|
||||
}
|
||||
public float getRight() {
|
||||
return (float) this.getMaxX();
|
||||
}
|
||||
|
||||
public void setTop(float top) {
|
||||
float deltaHeight = top - this.y;
|
||||
this.setRect(this.x, top, this.width, this.height - deltaHeight);
|
||||
}
|
||||
public void setRight(float right) {
|
||||
this.setRect(this.x, this.y, right - this.x, this.height);
|
||||
}
|
||||
|
||||
public float getRight() {
|
||||
return (float) this.getMaxX();
|
||||
}
|
||||
public float getLeft() {
|
||||
return (float) this.getMinX();
|
||||
}
|
||||
|
||||
public void setRight(float right) {
|
||||
this.setRect(this.x, this.y, right - this.x, this.height);
|
||||
}
|
||||
public void setLeft(float left) {
|
||||
float deltaWidth = left - this.x;
|
||||
this.setRect(left, this.y, this.width - deltaWidth, this.height);
|
||||
}
|
||||
|
||||
public float getLeft() {
|
||||
return (float) this.getMinX();
|
||||
}
|
||||
public float getBottom() {
|
||||
return (float) this.getMaxY();
|
||||
}
|
||||
|
||||
public void setLeft(float left) {
|
||||
float deltaWidth = left - this.x;
|
||||
this.setRect(left, this.y, this.width - deltaWidth, this.height);
|
||||
}
|
||||
public void setBottom(float bottom) {
|
||||
this.setRect(this.x, this.y, this.width, bottom - this.y);
|
||||
}
|
||||
|
||||
public float getBottom() {
|
||||
return (float) this.getMaxY();
|
||||
}
|
||||
public Point2D[] getPoints() {
|
||||
return new Point2D[] { new Point2D.Float(this.getLeft(), this.getTop()),
|
||||
new Point2D.Float(this.getRight(), this.getTop()), new Point2D.Float(this.getRight(), this.getBottom()),
|
||||
new Point2D.Float(this.getLeft(), this.getBottom()) };
|
||||
}
|
||||
|
||||
public void setBottom(float bottom) {
|
||||
this.setRect(this.x, this.y, this.width, bottom - this.y);
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String s = super.toString();
|
||||
sb.append(s.substring(0, s.length() - 1));
|
||||
sb.append(String.format(",bottom=%f,right=%f]", this.getBottom(), this.getRight()));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public Point2D[] getPoints() {
|
||||
return new Point2D[]{new Point2D.Float(this.getLeft(), this.getTop()),
|
||||
new Point2D.Float(this.getRight(), this.getTop()), new Point2D.Float(this.getRight(), this.getBottom()),
|
||||
new Point2D.Float(this.getLeft(), this.getBottom())};
|
||||
}
|
||||
/**
|
||||
* @param rectangles
|
||||
* @return minimum bounding box that contains all the rectangles
|
||||
*/
|
||||
public static Rectangle boundingBoxOf(List<? extends Rectangle> rectangles) {
|
||||
float minx = java.lang.Float.MAX_VALUE;
|
||||
float miny = java.lang.Float.MAX_VALUE;
|
||||
float maxx = java.lang.Float.MIN_VALUE;
|
||||
float maxy = java.lang.Float.MIN_VALUE;
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String s = super.toString();
|
||||
sb.append(s.substring(0, s.length() - 1));
|
||||
sb.append(String.format(",bottom=%f,right=%f]", this.getBottom(), this.getRight()));
|
||||
return sb.toString();
|
||||
}
|
||||
for (Rectangle r : rectangles) {
|
||||
minx = (float) Math.min(r.getMinX(), minx);
|
||||
miny = (float) Math.min(r.getMinY(), miny);
|
||||
maxx = (float) Math.max(r.getMaxX(), maxx);
|
||||
maxy = (float) Math.max(r.getMaxY(), maxy);
|
||||
}
|
||||
return new Rectangle(miny, minx, maxx - minx, maxy - miny);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-4
@@ -1,12 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.index.strtree.STRtree;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.locationtech.jts.geom.Envelope;
|
||||
import org.locationtech.jts.index.strtree.STRtree;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
@SuppressWarnings("all")
|
||||
public class RectangleSpatialIndex<T extends Rectangle> {
|
||||
|
||||
|
||||
+234
-226
@@ -1,13 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.server.tableextraction.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Formatter;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.CohenSutherlandClipping;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
@Slf4j
|
||||
@@ -16,10 +23,231 @@ public class Ruling extends Line2D.Float {
|
||||
|
||||
private static int PERPENDICULAR_PIXEL_EXPAND_AMOUNT = 2;
|
||||
|
||||
private enum SOType {VERTICAL, HRIGHT, HLEFT}
|
||||
|
||||
|
||||
public Ruling(Point2D p1, Point2D p2) {
|
||||
super(p1, p2);
|
||||
}
|
||||
|
||||
public boolean vertical() {
|
||||
return this.length() > 0 && Utils.feq(this.x1, this.x2); //diff < ORIENTATION_CHECK_THRESHOLD;
|
||||
}
|
||||
|
||||
public boolean horizontal() {
|
||||
return this.length() > 0 && Utils.feq(this.y1, this.y2); //diff < ORIENTATION_CHECK_THRESHOLD;
|
||||
}
|
||||
|
||||
public boolean oblique() {
|
||||
return !(this.vertical() || this.horizontal());
|
||||
}
|
||||
|
||||
// attributes that make sense only for non-oblique lines
|
||||
// these are used to have a single collapse method (in page, currently)
|
||||
|
||||
public float getPosition() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getLeft() : this.getTop();
|
||||
}
|
||||
|
||||
|
||||
public float getStart() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getTop() : this.getLeft();
|
||||
}
|
||||
|
||||
public void setStart(float v) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setTop(v);
|
||||
} else {
|
||||
this.setLeft(v);
|
||||
}
|
||||
}
|
||||
|
||||
public float getEnd() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getBottom() : this.getRight();
|
||||
}
|
||||
|
||||
public void setEnd(float v) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setBottom(v);
|
||||
} else {
|
||||
this.setRight(v);
|
||||
}
|
||||
}
|
||||
|
||||
public void setStartEnd(float start, float end) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setTop(start);
|
||||
this.setBottom(end);
|
||||
} else {
|
||||
this.setLeft(start);
|
||||
this.setRight(end);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean perpendicularTo(Ruling other) {
|
||||
return this.vertical() == other.horizontal();
|
||||
}
|
||||
|
||||
|
||||
public boolean nearlyIntersects(Ruling another, int colinearOrParallelExpandAmount) {
|
||||
if (this.intersectsLine(another)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean rv = false;
|
||||
|
||||
if (this.perpendicularTo(another)) {
|
||||
rv = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another);
|
||||
} else {
|
||||
rv = this.expand(colinearOrParallelExpandAmount)
|
||||
.intersectsLine(another.expand(colinearOrParallelExpandAmount));
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
public double length() {
|
||||
return Math.sqrt(Math.pow(this.x1 - this.x2, 2) + Math.pow(this.y1 - this.y2, 2));
|
||||
}
|
||||
|
||||
public Ruling intersect(Rectangle2D clip) {
|
||||
Float clipee = (Float) this.clone();
|
||||
boolean clipped = new CohenSutherlandClipping(clip).clip(clipee);
|
||||
|
||||
if (clipped) {
|
||||
return new Ruling(clipee.getP1(), clipee.getP2());
|
||||
} else {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
public Ruling expand(float amount) {
|
||||
Ruling r = (Ruling) this.clone();
|
||||
r.setStart(this.getStart() - amount);
|
||||
r.setEnd(this.getEnd() + amount);
|
||||
return r;
|
||||
}
|
||||
|
||||
public Point2D intersectionPoint(Ruling other) {
|
||||
Ruling this_l = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
|
||||
Ruling other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
|
||||
Ruling horizontal, vertical;
|
||||
|
||||
if (!this_l.intersectsLine(other_l)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this_l.horizontal() && other_l.vertical()) {
|
||||
horizontal = this_l;
|
||||
vertical = other_l;
|
||||
} else if (this_l.vertical() && other_l.horizontal()) {
|
||||
vertical = this_l;
|
||||
horizontal = other_l;
|
||||
} else {
|
||||
log.warn("lines must be orthogonal, vertical and horizontal");
|
||||
return null;
|
||||
}
|
||||
return new Point2D.Float(vertical.getLeft(), horizontal.getTop());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (this == other) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!(other instanceof Ruling)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Ruling o = (Ruling) other;
|
||||
return this.getP1().equals(o.getP1()) && this.getP2().equals(o.getP2());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return super.hashCode();
|
||||
}
|
||||
|
||||
public float getTop() {
|
||||
return this.y1;
|
||||
}
|
||||
|
||||
public void setTop(float v) {
|
||||
setLine(this.getLeft(), v, this.getRight(), this.getBottom());
|
||||
}
|
||||
|
||||
public float getLeft() {
|
||||
return this.x1;
|
||||
}
|
||||
|
||||
public void setLeft(float v) {
|
||||
setLine(v, this.getTop(), this.getRight(), this.getBottom());
|
||||
}
|
||||
|
||||
public float getBottom() {
|
||||
return this.y2;
|
||||
}
|
||||
|
||||
public void setBottom(float v) {
|
||||
setLine(this.getLeft(), this.getTop(), this.getRight(), v);
|
||||
}
|
||||
|
||||
public float getRight() {
|
||||
return this.x2;
|
||||
}
|
||||
|
||||
public void setRight(float v) {
|
||||
setLine(this.getLeft(), this.getTop(), v, this.getBottom());
|
||||
}
|
||||
|
||||
public float getWidth() {
|
||||
return this.getRight() - this.getLeft();
|
||||
}
|
||||
|
||||
public float getHeight() {
|
||||
return this.getBottom() - this.getTop();
|
||||
}
|
||||
|
||||
public double getAngle() {
|
||||
double angle = Math.toDegrees(Math.atan2(this.getP2().getY() - this.getP1().getY(),
|
||||
this.getP2().getX() - this.getP1().getX()));
|
||||
|
||||
if (angle < 0) {
|
||||
angle += 360;
|
||||
}
|
||||
return angle;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Formatter formatter = new Formatter(sb);
|
||||
String rv = formatter.format("%s[minX=%f minY=%f maxX=%f maxY=%f]", this.getClass().toString(), this.x1, this.y1, this.x2, this.y2).toString();
|
||||
formatter.close();
|
||||
return rv;
|
||||
}
|
||||
|
||||
public static List<Ruling> cropRulingsToArea(List<Ruling> rulings, Rectangle2D area) {
|
||||
ArrayList<Ruling> rv = new ArrayList<>();
|
||||
for (Ruling r : rulings) {
|
||||
@@ -118,7 +346,7 @@ public class Ruling extends Line2D.Float {
|
||||
rv.put(i,
|
||||
new Ruling[]{h.getKey().expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT),
|
||||
so.ruling.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)});
|
||||
} catch (UnsupportedOperationException e) {
|
||||
} catch(UnsupportedOperationException e){
|
||||
log.info("Some line are oblique, ignoring...");
|
||||
continue;
|
||||
}
|
||||
@@ -137,225 +365,5 @@ public class Ruling extends Line2D.Float {
|
||||
|
||||
}
|
||||
|
||||
public boolean vertical() {
|
||||
return this.length() > 0 && Utils.feq(this.x1, this.x2); //diff < ORIENTATION_CHECK_THRESHOLD;
|
||||
}
|
||||
|
||||
public boolean horizontal() {
|
||||
return this.length() > 0 && Utils.feq(this.y1, this.y2); //diff < ORIENTATION_CHECK_THRESHOLD;
|
||||
}
|
||||
|
||||
// attributes that make sense only for non-oblique lines
|
||||
// these are used to have a single collapse method (in page, currently)
|
||||
|
||||
public boolean oblique() {
|
||||
return !(this.vertical() || this.horizontal());
|
||||
}
|
||||
|
||||
public float getPosition() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getLeft() : this.getTop();
|
||||
}
|
||||
|
||||
public float getStart() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getTop() : this.getLeft();
|
||||
}
|
||||
|
||||
public void setStart(float v) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setTop(v);
|
||||
} else {
|
||||
this.setLeft(v);
|
||||
}
|
||||
}
|
||||
|
||||
public float getEnd() {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
return this.vertical() ? this.getBottom() : this.getRight();
|
||||
}
|
||||
|
||||
public void setEnd(float v) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setBottom(v);
|
||||
} else {
|
||||
this.setRight(v);
|
||||
}
|
||||
}
|
||||
|
||||
public void setStartEnd(float start, float end) {
|
||||
if (this.oblique()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
if (this.vertical()) {
|
||||
this.setTop(start);
|
||||
this.setBottom(end);
|
||||
} else {
|
||||
this.setLeft(start);
|
||||
this.setRight(end);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean perpendicularTo(Ruling other) {
|
||||
return this.vertical() == other.horizontal();
|
||||
}
|
||||
|
||||
public boolean nearlyIntersects(Ruling another, int colinearOrParallelExpandAmount) {
|
||||
if (this.intersectsLine(another)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean rv = false;
|
||||
|
||||
if (this.perpendicularTo(another)) {
|
||||
rv = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another);
|
||||
} else {
|
||||
rv = this.expand(colinearOrParallelExpandAmount)
|
||||
.intersectsLine(another.expand(colinearOrParallelExpandAmount));
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
public double length() {
|
||||
return Math.sqrt(Math.pow(this.x1 - this.x2, 2) + Math.pow(this.y1 - this.y2, 2));
|
||||
}
|
||||
|
||||
public Ruling intersect(Rectangle2D clip) {
|
||||
Float clipee = (Float) this.clone();
|
||||
boolean clipped = new CohenSutherlandClipping(clip).clip(clipee);
|
||||
|
||||
if (clipped) {
|
||||
return new Ruling(clipee.getP1(), clipee.getP2());
|
||||
} else {
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
public Ruling expand(float amount) {
|
||||
Ruling r = (Ruling) this.clone();
|
||||
try {
|
||||
r.setStart(this.getStart() - amount);
|
||||
r.setEnd(this.getEnd() + amount);
|
||||
} catch (UnsupportedOperationException e){
|
||||
log.warn("Could not expand ruling!");
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
public Point2D intersectionPoint(Ruling other) {
|
||||
Ruling this_l = this.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
|
||||
Ruling other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT);
|
||||
Ruling horizontal, vertical;
|
||||
|
||||
if (!this_l.intersectsLine(other_l)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (this_l.horizontal() && other_l.vertical()) {
|
||||
horizontal = this_l;
|
||||
vertical = other_l;
|
||||
} else if (this_l.vertical() && other_l.horizontal()) {
|
||||
vertical = this_l;
|
||||
horizontal = other_l;
|
||||
} else {
|
||||
log.warn("lines must be orthogonal, vertical and horizontal");
|
||||
return null;
|
||||
}
|
||||
return new Point2D.Float(vertical.getLeft(), horizontal.getTop());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object other) {
|
||||
if (this == other) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!(other instanceof Ruling)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Ruling o = (Ruling) other;
|
||||
return this.getP1().equals(o.getP1()) && this.getP2().equals(o.getP2());
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return super.hashCode();
|
||||
}
|
||||
|
||||
public float getTop() {
|
||||
return this.y1;
|
||||
}
|
||||
|
||||
public void setTop(float v) {
|
||||
setLine(this.getLeft(), v, this.getRight(), this.getBottom());
|
||||
}
|
||||
|
||||
public float getLeft() {
|
||||
return this.x1;
|
||||
}
|
||||
|
||||
public void setLeft(float v) {
|
||||
setLine(v, this.getTop(), this.getRight(), this.getBottom());
|
||||
}
|
||||
|
||||
public float getBottom() {
|
||||
return this.y2;
|
||||
}
|
||||
|
||||
public void setBottom(float v) {
|
||||
setLine(this.getLeft(), this.getTop(), this.getRight(), v);
|
||||
}
|
||||
|
||||
public float getRight() {
|
||||
return this.x2;
|
||||
}
|
||||
|
||||
public void setRight(float v) {
|
||||
setLine(this.getLeft(), this.getTop(), v, this.getBottom());
|
||||
}
|
||||
|
||||
public float getWidth() {
|
||||
return this.getRight() - this.getLeft();
|
||||
}
|
||||
|
||||
public float getHeight() {
|
||||
return this.getBottom() - this.getTop();
|
||||
}
|
||||
|
||||
public double getAngle() {
|
||||
double angle = Math.toDegrees(Math.atan2(this.getP2().getY() - this.getP1().getY(),
|
||||
this.getP2().getX() - this.getP1().getX()));
|
||||
|
||||
if (angle < 0) {
|
||||
angle += 360;
|
||||
}
|
||||
return angle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
Formatter formatter = new Formatter(sb);
|
||||
String rv = formatter.format("%s[minX=%f minY=%f maxX=%f maxY=%f]", this.getClass().toString(), this.x1, this.y1, this.x2, this.y2).toString();
|
||||
formatter.close();
|
||||
return rv;
|
||||
}
|
||||
|
||||
private enum SOType {VERTICAL, HRIGHT, HLEFT}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user