Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 deiflaender
|
c9645c3329 |
@@ -1,5 +1,4 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
+28
-28
@@ -1,37 +1,37 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-parent</artifactId>
|
||||
<version>8.1.3</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
<parent>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-parent</artifactId>
|
||||
<version>7.1.2</version>
|
||||
<relativePath/>
|
||||
</parent>
|
||||
|
||||
<artifactId>bamboo-specs</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
<artifactId>bamboo-specs</artifactId>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs</artifactId>
|
||||
</dependency>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.atlassian.bamboo</groupId>
|
||||
<artifactId>bamboo-specs</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<!-- Test dependencies -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
<!-- run 'mvn test' to perform offline validation of the plan -->
|
||||
<!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
|
||||
<!-- run 'mvn test' to perform offline validation of the plan -->
|
||||
<!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
|
||||
</project>
|
||||
|
||||
@@ -2,11 +2,8 @@ package buildjob;
|
||||
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
|
||||
import java.time.LocalTime;
|
||||
|
||||
import com.atlassian.bamboo.specs.api.BambooSpec;
|
||||
import com.atlassian.bamboo.specs.api.builders.BambooKey;
|
||||
import com.atlassian.bamboo.specs.api.builders.Variable;
|
||||
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
|
||||
@@ -19,15 +16,12 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
|
||||
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
|
||||
import com.atlassian.bamboo.specs.api.builders.project.Project;
|
||||
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
|
||||
import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
|
||||
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
|
||||
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
|
||||
/**
|
||||
@@ -39,15 +33,12 @@ public class PlanSpec {
|
||||
|
||||
private static final String SERVICE_NAME = "redaction-service";
|
||||
|
||||
private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
|
||||
|
||||
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
|
||||
|
||||
|
||||
/**
|
||||
* Run main to publish plan on Bamboo
|
||||
*/
|
||||
public static void main(final String[] args) {
|
||||
public static void main(final String[] args) throws Exception {
|
||||
//By default credentials are read from the '.credentials' file.
|
||||
BambooServer bambooServer = new BambooServer("http://localhost:8085");
|
||||
|
||||
@@ -55,102 +46,92 @@ public class PlanSpec {
|
||||
bambooServer.publish(plan);
|
||||
PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
|
||||
bambooServer.publish(planPermission);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
bambooServer.publish(nightPlan);
|
||||
PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
|
||||
bambooServer.publish(nightPlanPermission);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
bambooServer.publish(secPlan);
|
||||
PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
|
||||
bambooServer.publish(secPlanPermission);
|
||||
}
|
||||
|
||||
|
||||
private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
|
||||
|
||||
Permissions permission = new Permissions().userPermissions("atlbamboo",
|
||||
PermissionType.EDIT,
|
||||
PermissionType.VIEW,
|
||||
PermissionType.ADMIN,
|
||||
PermissionType.CLONE,
|
||||
PermissionType.BUILD)
|
||||
.groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
Permissions permission = new Permissions()
|
||||
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("red-backend", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.loggedInUserPermissions(PermissionType.VIEW)
|
||||
.anonymousUserPermissionView();
|
||||
return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
|
||||
}
|
||||
|
||||
|
||||
private Project project() {
|
||||
|
||||
return new Project().name("RED").key(new BambooKey("RED"));
|
||||
return new Project()
|
||||
.name("RED")
|
||||
.key(new BambooKey("RED"));
|
||||
}
|
||||
|
||||
|
||||
public Plan createPlan() {
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME, new BambooKey(SERVICE_KEY))
|
||||
.description("Plan created from (enter repository url of your plan)")
|
||||
.stages(new Stage("Default Stage")
|
||||
.jobs(new Job("Default Job",
|
||||
new BambooKey("JOB1"))
|
||||
.tasks(
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
"rm -rf ./*"),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout Default Repository")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
"rm -rf ./*"),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout Default Repository")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Build")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
|
||||
return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
|
||||
.variables(new Variable("maven_add_param", ""))
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
|
||||
.enabled(true),
|
||||
new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask().description("Build").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh").argument(SERVICE_NAME),
|
||||
createJUnitParserTask().description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true),
|
||||
new InjectVariablesTask().description("Inject git Tag").path("git.tag").namespace("g").scope(InjectVariablesScope.LOCAL),
|
||||
new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml versions:set -DnewVersion=${bamboo.version_tag}; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml versions:set -DnewVersion=${bamboo.version_tag}; fi\n" +
|
||||
|
||||
"if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean install -Djava.security.egd=file:/dev/./urandom; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-v1/pom.xml --no-transfer-progress clean deploy -e -DdeployAtEnd=true -Dmaven.wagon.http.ssl.insecure=true -Dmaven.wagon.http.ssl.allowall=true -Dmaven.wagon.http.ssl.ignore.validity.dates=true -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases; fi\n" +
|
||||
|
||||
"${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml package\n" +
|
||||
"${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn --no-transfer-progress -f ${bamboo_build_working_directory}/" + SERVICE_NAME + "-image-v1/pom.xml docker:push\n" +
|
||||
|
||||
"if [[ \"${bamboo.version_tag}\" = \"dev\" ]]; then echo \"gitTag=${bamboo.planRepository.1.branch}_${bamboo.buildNumber}\" > git.tag; fi\n" +
|
||||
"if [[ \"${bamboo.version_tag}\" != \"dev\" ]]; then echo \"gitTag=${bamboo.version_tag}\" > git.tag; fi\n"),
|
||||
createJUnitParserTask()
|
||||
.description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true),
|
||||
new InjectVariablesTask()
|
||||
.description("Inject git Tag")
|
||||
.path("git.tag")
|
||||
.namespace("g")
|
||||
.scope(InjectVariablesScope.LOCAL),
|
||||
new VcsTagTask()
|
||||
.description("${bamboo.g.gitTag}")
|
||||
.tagName("${bamboo.g.gitTag}")
|
||||
.defaultRepository())
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock")
|
||||
)
|
||||
)
|
||||
)
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
|
||||
.triggers(new BitbucketServerTrigger())
|
||||
.planBranchManagement(new PlanBranchManagement().createForVcsBranch()
|
||||
.delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
|
||||
.planBranchManagement(new PlanBranchManagement()
|
||||
.createForVcsBranch()
|
||||
.delete(new BranchCleanup()
|
||||
.whenInactiveInRepositoryAfterDays(14))
|
||||
.notificationForCommitters());
|
||||
}
|
||||
|
||||
|
||||
public Plan createNightPlan() {
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
|
||||
.variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
|
||||
.enabled(true),
|
||||
new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask().description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME + " verify"),
|
||||
createJUnitParserTask().description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true))
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
|
||||
}
|
||||
|
||||
|
||||
public Plan createSecBuild() {
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"),
|
||||
new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true).checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask().description("Sonar").location(Location.FILE).fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh").argument(SERVICE_NAME))
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.dockerRunArguments("--net=host")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
SERVICE_NAME=$1
|
||||
MVN_TARGET=${2:-deploy}
|
||||
|
||||
if [[ "$bamboo_planRepository_branchName" == "master" ]]
|
||||
then
|
||||
branchVersion=$(cat pom.xml | grep -Eo " <version>.*-SNAPSHOT</version>" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|' | tr -d ' ')
|
||||
latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
|
||||
newVersion="$(semver $latestVersion -p -i minor)"
|
||||
echo "new release on master with version $newVersion"
|
||||
elif [[ "$bamboo_planRepository_branchName" == release* ]]
|
||||
then
|
||||
branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
|
||||
latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
|
||||
newVersion="$(semver $latestVersion -p -i patch)"
|
||||
echo "new release on $bamboo_planRepository_branchName with version $newVersion"
|
||||
elif [[ "${bamboo_version_tag}" != "dev" ]]
|
||||
then
|
||||
newVersion="${bamboo_version_tag}"
|
||||
echo "new special version bild with $newVersion"
|
||||
else
|
||||
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
${bamboo_maven_add_param} \
|
||||
clean install \
|
||||
-Djava.security.egd=file:/dev/./urandomelse
|
||||
echo "dev build with tag ${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
|
||||
echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "gitTag=${newVersion}" > git.tag
|
||||
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
${bamboo_maven_add_param} \
|
||||
versions:set \
|
||||
-DnewVersion=${newVersion}
|
||||
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
${bamboo_maven_add_param} \
|
||||
versions:set \
|
||||
-DnewVersion=${newVersion}
|
||||
|
||||
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
clean $MVN_TARGET \
|
||||
${bamboo_maven_add_param} \
|
||||
-e \
|
||||
-DdeployAtEnd=true \
|
||||
-Dmaven.wagon.http.ssl.insecure=true \
|
||||
-Dmaven.wagon.http.ssl.allowall=true \
|
||||
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
|
||||
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
|
||||
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
package
|
||||
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
|
||||
docker:push
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
SERVICE_NAME=$1
|
||||
|
||||
echo "build jar binaries"
|
||||
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
clean install \
|
||||
-Djava.security.egd=file:/dev/./urandomelse
|
||||
|
||||
echo "dependency-check:aggregate"
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
org.owasp:dependency-check-maven:aggregate \
|
||||
-DknownExploitedEnabled=false
|
||||
|
||||
if [[ -z "${bamboo_repository_pr_key}" ]]
|
||||
then
|
||||
echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
sonar:sonar \
|
||||
-Dsonar.projectKey=RED_$SERVICE_NAME \
|
||||
-Dsonar.host.url=https://sonarqube.iqser.com \
|
||||
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
|
||||
-Dsonar.branch.name=${bamboo_planRepository_1_branch} \
|
||||
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
|
||||
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
|
||||
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
|
||||
else
|
||||
echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
|
||||
mvn --no-transfer-progress \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
sonar:sonar \
|
||||
-Dsonar.projectKey=RED_$SERVICE_NAME \
|
||||
-Dsonar.host.url=https://sonarqube.iqser.com \
|
||||
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
|
||||
-Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
|
||||
-Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
|
||||
-Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
|
||||
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
|
||||
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
|
||||
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
|
||||
fi
|
||||
@@ -1,5 +1,6 @@
|
||||
package buildjob;
|
||||
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import com.atlassian.bamboo.specs.api.builders.plan.Plan;
|
||||
@@ -7,18 +8,10 @@ import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
|
||||
import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
|
||||
|
||||
public class PlanSpecTest {
|
||||
|
||||
@Test
|
||||
public void checkYourPlanOffline() throws PropertiesValidationException {
|
||||
|
||||
Plan plan = new PlanSpec().createPlan();
|
||||
|
||||
EntityPropertiesBuilders.build(plan);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
EntityPropertiesBuilders.build(nightPlan);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
EntityPropertiesBuilders.build(secPlan);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
<artifactId>redaction-service</artifactId>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<version>4.0-SNAPSHOT</version>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
|
||||
<packaging>pom</packaging>
|
||||
@@ -18,4 +18,4 @@
|
||||
<module>redaction-service-image-v1</module>
|
||||
</modules>
|
||||
|
||||
</project>
|
||||
</project>
|
||||
@@ -5,8 +5,7 @@
|
||||
<parent>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-docker-dependency</artifactId>
|
||||
<version>1.2.0</version>
|
||||
<relativePath/>
|
||||
<version>1.0.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
@@ -42,7 +41,7 @@
|
||||
<artifactId>docker-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
@@ -95,4 +94,4 @@
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM red/redaction-service-base-v1:2.0.0
|
||||
FROM red/base-image:1.0.0
|
||||
|
||||
ARG PLATFORM_JAR
|
||||
|
||||
@@ -7,3 +7,12 @@ ENV PLATFORM_JAR ${PLATFORM_JAR}
|
||||
ENV USES_ELASTICSEARCH false
|
||||
|
||||
COPY ["${PLATFORM_JAR}", "/"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
wget cabextract xfonts-utils fonts-liberation \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl http://ftp.br.debian.org/debian/pool/contrib/m/msttcorefonts/ttf-mscorefonts-installer_3.7_all.deb -o /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
&& dpkg -i /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
&& rm /tmp/ttf-mscorefonts-installer_3.7_all.deb \
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
<parent>
|
||||
<artifactId>platform-dependency</artifactId>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<version>1.17.0</version>
|
||||
<relativePath/>
|
||||
<version>1.0.2</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>redaction-service-v1</artifactId>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<version>4.0-SNAPSHOT</version>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
|
||||
<packaging>pom</packaging>
|
||||
|
||||
@@ -22,20 +22,21 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<pdfbox.version>2.0.24</pdfbox.version>
|
||||
<dsljson.version>1.9.9</dsljson.version>
|
||||
<pdfbox.version>2.0.21</pdfbox.version>
|
||||
</properties>
|
||||
|
||||
|
||||
<dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.22.0</version>
|
||||
<version>1.1.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox</artifactId>
|
||||
@@ -46,48 +47,9 @@
|
||||
<artifactId>pdfbox-tools</artifactId>
|
||||
<version>${pdfbox.version}</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</dependencyManagement>
|
||||
|
||||
<build>
|
||||
<pluginManagement>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.sonarsource.scanner.maven</groupId>
|
||||
<artifactId>sonar-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.owasp</groupId>
|
||||
<artifactId>dependency-check-maven</artifactId>
|
||||
<configuration>
|
||||
<format>ALL</format>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.jacoco</groupId>
|
||||
<artifactId>jacoco-maven-plugin</artifactId>
|
||||
<version>0.8.8</version>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>org/drools/**/*</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>prepare-agent</id>
|
||||
<goals>
|
||||
<goal>prepare-agent</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>report</id>
|
||||
<goals>
|
||||
<goal>report</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</pluginManagement>
|
||||
</build>
|
||||
</project>
|
||||
</project>
|
||||
@@ -6,61 +6,16 @@
|
||||
<parent>
|
||||
<artifactId>redaction-service-v1</artifactId>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<version>4.0-SNAPSHOT</version>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<persistence-service.version>2.1.0</persistence-service.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/com.dslplatform/dsl-json-java8 -->
|
||||
<dependency>
|
||||
<groupId>com.dslplatform</groupId>
|
||||
<artifactId>dsl-json-java8</artifactId>
|
||||
<version>${dsljson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-web</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>persistence-service-internal-api-v1</artifactId>
|
||||
<version>${persistence-service.version}</version>
|
||||
<exclusions>
|
||||
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>persistence-service-api-v1</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<annotationProcessors>
|
||||
<annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
|
||||
<annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
|
||||
</annotationProcessors>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ArgumentType {
|
||||
|
||||
INTEGER,
|
||||
BOOLEAN,
|
||||
STRING,
|
||||
FILE_ATTRIBUTE,
|
||||
REGEX,
|
||||
TYPE,
|
||||
RULE_NUMBER,
|
||||
LEGAL_BASIS,
|
||||
REFERENCE_TYPE
|
||||
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class CellRectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
}
|
||||
+7
-3
@@ -1,4 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client.model;
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@@ -9,9 +11,11 @@ import lombok.NoArgsConstructor;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class EntityRecognitionSection {
|
||||
public class Comment {
|
||||
|
||||
private int sectionNumber;
|
||||
private String id;
|
||||
private OffsetDateTime date;
|
||||
private String text;
|
||||
private String user;
|
||||
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class IdRemoval {
|
||||
|
||||
private String id;
|
||||
private String user;
|
||||
private Status status;
|
||||
private boolean removeFromDictionary;
|
||||
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class ManualRedactionEntry {
|
||||
|
||||
private String id;
|
||||
private String user;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private String legalBasis;
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private Status status;
|
||||
private boolean addToDictionary;
|
||||
|
||||
private String section;
|
||||
private int sectionNumber;
|
||||
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ManualRedactionType {
|
||||
ADD, REMOVE
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class ManualRedactions {
|
||||
|
||||
@Builder.Default
|
||||
private Set<IdRemoval> idsToRemove = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Map<String, List<Comment>> comments = new HashMap<>();
|
||||
|
||||
}
|
||||
+4
-4
@@ -5,11 +5,11 @@ import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Argument {
|
||||
@NoArgsConstructor
|
||||
public class Point {
|
||||
|
||||
private String name;
|
||||
private ArgumentType type;
|
||||
private float x;
|
||||
private float y;
|
||||
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class Rectangle {
|
||||
|
||||
private Point topLeft;
|
||||
private float width;
|
||||
private float height;
|
||||
|
||||
private int page;
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class RedactionLog {
|
||||
|
||||
public RedactionLog(List<RedactionLogEntry> redactionLogEntry, long dictionaryVersion, long rulesVersion) {
|
||||
|
||||
this.redactionLogEntry = redactionLogEntry;
|
||||
this.dictionaryVersion = dictionaryVersion;
|
||||
this.rulesVersion = rulesVersion;
|
||||
}
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
|
||||
private long dictionaryVersion = -1;
|
||||
|
||||
private long rulesVersion = -1;
|
||||
|
||||
private String filename;
|
||||
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RedactionLogEntry {
|
||||
|
||||
private String id;
|
||||
private String type;
|
||||
private String value;
|
||||
private String reason;
|
||||
private int matchedRule;
|
||||
private String legalBasis;
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
private boolean manual;
|
||||
private Status status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
|
||||
}
|
||||
+4
-5
@@ -9,10 +9,9 @@ import lombok.NoArgsConstructor;
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class StructureAnalyzeRequest {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
public class RedactionRequest {
|
||||
|
||||
private byte[] document;
|
||||
private boolean flatRedaction;
|
||||
private ManualRedactions manualRedactions;
|
||||
}
|
||||
|
||||
+5
-2
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.annotate;
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
@@ -9,8 +9,11 @@ import lombok.NoArgsConstructor;
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnnotateResponse {
|
||||
public class RedactionResult {
|
||||
|
||||
private byte[] document;
|
||||
private int numberOfPages;
|
||||
private RedactionLog redactionLog;
|
||||
private SectionGrid sectionGrid;
|
||||
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
public class RuleBuilderModel {
|
||||
|
||||
private List<RuleElement> whenClauses = new ArrayList<>();
|
||||
private List<RuleElement> thenConditions = new ArrayList<>();
|
||||
|
||||
}
|
||||
+7
-7
@@ -1,18 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class RuleElement {
|
||||
@NoArgsConstructor
|
||||
public class SectionGrid {
|
||||
|
||||
private String conditionName;
|
||||
private List<Argument> arguments = new ArrayList<>();
|
||||
private Map<Integer, List<SectionRectangle>> rectanglesPerPage = new HashMap<>();
|
||||
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public class SectionRectangle {
|
||||
|
||||
@NonNull
|
||||
private Point topLeft;
|
||||
|
||||
@NonNull
|
||||
private float width;
|
||||
|
||||
@NonNull
|
||||
private float height;
|
||||
|
||||
@NonNull
|
||||
private int part;
|
||||
|
||||
@NonNull
|
||||
private int numberOfParts;
|
||||
|
||||
private List<CellRectangle> tableCells;
|
||||
}
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum Status {
|
||||
REQUESTED, APPROVED, DECLINED
|
||||
}
|
||||
+20
-3
@@ -4,9 +4,26 @@ import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void testRules(@RequestBody String rules);
|
||||
String SERVICE_NAME = "redaction-service-v1";
|
||||
|
||||
}
|
||||
@PostMapping(value = "/redact", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult redact(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/classifications", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult classify(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/sections", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult sections(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/debug/htmlTables", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest);
|
||||
|
||||
@PostMapping(value = "/rules/update", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
void updateRules(@RequestBody String rules);
|
||||
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.resources;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
|
||||
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
|
||||
public interface RuleBuilderResource {
|
||||
|
||||
@PostMapping(value = "/rule-builder-model", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
RuleBuilderModel getRuleBuilderModel();
|
||||
|
||||
}
|
||||
@@ -6,81 +6,42 @@
|
||||
<parent>
|
||||
<artifactId>redaction-service-v1</artifactId>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<version>4.0-SNAPSHOT</version>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>redaction-service-server-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<drools.version>7.73.0.Final</drools.version>
|
||||
<kie.version>7.73.0.Final</kie.version>
|
||||
<locationtech.version>1.18.2</locationtech.version>
|
||||
<javaassist.version>3.28.0-GA</javaassist.version>
|
||||
<ahocorasick.version>0.6.3</ahocorasick.version>
|
||||
<jackson.version>2.13.2</jackson.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-aop</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<artifactId>storage-commons</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.module</groupId>
|
||||
<artifactId>jackson-module-afterburner</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
<version>${jackson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.ahocorasick</groupId>
|
||||
<artifactId>ahocorasick</artifactId>
|
||||
<version>${ahocorasick.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.javassist</groupId>
|
||||
<artifactId>javassist</artifactId>
|
||||
<version>${javaassist.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.service</groupId>
|
||||
<artifactId>configuration-service-api-v1</artifactId>
|
||||
<version>1.3.7</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.drools</groupId>
|
||||
<artifactId>drools-core</artifactId>
|
||||
<version>${drools.version}</version>
|
||||
<version>7.37.0.Final</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.kie</groupId>
|
||||
<artifactId>kie-spring</artifactId>
|
||||
<version>${kie.version}</version>
|
||||
<version>7.37.0.Final</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.locationtech.jts</groupId>
|
||||
<artifactId>jts-core</artifactId>
|
||||
<version>${locationtech.version}</version>
|
||||
<version>1.16.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
<version>29.0-jre</version>
|
||||
</dependency>
|
||||
|
||||
<!-- commons -->
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
@@ -93,9 +54,12 @@
|
||||
<dependency>
|
||||
<groupId>com.iqser.red.commons</groupId>
|
||||
<artifactId>metric-commons</artifactId>
|
||||
<version>1.7.0</version>
|
||||
</dependency>
|
||||
<!-- other external -->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.pdfbox</groupId>
|
||||
<artifactId>pdfbox</artifactId>
|
||||
@@ -109,12 +73,11 @@
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-openfeign</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- ribbon must be included because transitive a dependency on it.-->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-amqp</artifactId>
|
||||
<groupId>org.springframework.cloud</groupId>
|
||||
<artifactId>spring-cloud-starter-netflix-ribbon</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- test dependencies -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -130,18 +93,6 @@
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<annotationProcessors>
|
||||
<annotationProcessor>lombok.launch.AnnotationProcessorHider$AnnotationProcessor</annotationProcessor>
|
||||
<annotationProcessor>com.dslplatform.json.processor.CompiledJsonAnnotationProcessor</annotationProcessor>
|
||||
</annotationProcessors>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<!-- generate git.properties for exposure in /info -->
|
||||
<groupId>pl.project13.maven</groupId>
|
||||
|
||||
+43
-12
@@ -1,6 +1,18 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.KieServices;
|
||||
import org.kie.api.builder.KieBuilder;
|
||||
import org.kie.api.builder.KieFileSystem;
|
||||
import org.kie.api.builder.KieModule;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.actuate.autoconfigure.metrics.web.servlet.WebMvcMetricsAutoConfiguration;
|
||||
import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
@@ -9,32 +21,51 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyWebConfiguration;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import io.micrometer.core.aop.TimedAspect;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
|
||||
@Import({MultiTenancyWebConfiguration.class, AsyncConfig.class, MultiTenancyMessagingConfiguration.class, MetricsConfiguration.class})
|
||||
@Import({DefaultWebMvcConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@EnableConfigurationProperties(RedactionServiceSettings.class)
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
|
||||
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, WebMvcMetricsAutoConfiguration.class})
|
||||
public class Application {
|
||||
|
||||
@Autowired
|
||||
private RulesClient rulesClient;
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
System.setProperty("org.apache.pdfbox.rendering.UsePureJavaCMYKConversion", "true");
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public TimedAspect timedAspect(MeterRegistry registry) {
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
try {
|
||||
KieServices kieServices = KieServices.Factory.get();
|
||||
|
||||
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
|
||||
RulesResponse rules = rulesClient.getRules();
|
||||
if (StringUtils.isEmpty(rules.getRules())) {
|
||||
throw new RuntimeException("Rules cannot be empty.");
|
||||
}
|
||||
InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
|
||||
kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
|
||||
.newInputStreamResource(input));
|
||||
KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
|
||||
kieBuilder.buildAll();
|
||||
KieModule kieModule = kieBuilder.getKieModule();
|
||||
|
||||
return kieServices.newKieContainer(kieModule.getReleaseId());
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
return new TimedAspect(registry);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.gin4.commons.metrics.MetricCommonsConfiguration;
|
||||
import com.iqser.gin4.commons.metrics.meters.FunctionTimerFactory;
|
||||
import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
|
||||
|
||||
@Configuration
|
||||
@Import({MetricCommonsConfiguration.class})
|
||||
public class MetricsConfiguration {
|
||||
|
||||
private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
|
||||
|
||||
|
||||
@Bean
|
||||
@Qualifier(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME)
|
||||
public FunctionTimerValues redactmanagerAnalyzePagewise(FunctionTimerFactory factory) {
|
||||
|
||||
return factory.create(REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME);
|
||||
}
|
||||
|
||||
}
|
||||
+10
-9
@@ -1,10 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
@@ -15,17 +18,15 @@ public class Document {
|
||||
|
||||
private List<Page> pages = new ArrayList<>();
|
||||
private List<Paragraph> paragraphs = new ArrayList<>();
|
||||
private List<Header> headers = new ArrayList<>();
|
||||
private List<Footer> footers = new ArrayList<>();
|
||||
private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
|
||||
private Map<Integer, List<Entity>> entities = new HashMap<>();
|
||||
private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter = new FloatFrequencyCounter();
|
||||
private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
|
||||
private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
|
||||
private StringFrequencyCounter fontCounter= new StringFrequencyCounter();
|
||||
private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
|
||||
private boolean headlines;
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
|
||||
private SectionGrid sectionGrid = new SectionGrid();
|
||||
private DictionaryVersion dictionaryVersion;
|
||||
private long dictionaryVersion;
|
||||
private long rulesVersion;
|
||||
|
||||
}
|
||||
|
||||
+23
-25
@@ -1,7 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
@@ -9,39 +7,38 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class FloatFrequencyCounter {
|
||||
import lombok.Getter;
|
||||
|
||||
public class FloatFrequencyCounter
|
||||
{
|
||||
|
||||
@Getter
|
||||
Map<Float, Integer> countPerValue = new HashMap<>();
|
||||
|
||||
|
||||
public void add(float value) {
|
||||
|
||||
if (!countPerValue.containsKey(value)) {
|
||||
public void add(float value){
|
||||
if(!countPerValue.containsKey(value)){
|
||||
countPerValue.put(value, 1);
|
||||
} else {
|
||||
countPerValue.put(value, countPerValue.get(value) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addAll(Map<Float, Integer> otherCounter) {
|
||||
|
||||
for (Map.Entry<Float, Integer> entry : otherCounter.entrySet()) {
|
||||
if (countPerValue.containsKey(entry.getKey())) {
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey()) + entry.getValue());
|
||||
public void addAll(Map<Float, Integer> otherCounter){
|
||||
for(Map.Entry<Float, Integer> entry: otherCounter.entrySet()){
|
||||
if(countPerValue.containsKey(entry.getKey())){
|
||||
countPerValue.put(entry.getKey(), countPerValue.get(entry.getKey())+ entry.getValue());
|
||||
} else {
|
||||
countPerValue.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Float getMostPopular() {
|
||||
|
||||
public Float getMostPopular(){
|
||||
Map.Entry<Float, Integer> mostPopular = null;
|
||||
for (Map.Entry<Float, Integer> entry : countPerValue.entrySet()) {
|
||||
if (mostPopular == null || entry.getValue() >= mostPopular.getValue()) {
|
||||
for(Map.Entry<Float, Integer> entry: countPerValue.entrySet()){
|
||||
if(mostPopular == null){
|
||||
mostPopular = entry;
|
||||
} else if(entry.getValue() >= mostPopular.getValue()){
|
||||
mostPopular = entry;
|
||||
}
|
||||
}
|
||||
@@ -49,12 +46,12 @@ public class FloatFrequencyCounter {
|
||||
}
|
||||
|
||||
|
||||
public List<Float> getHighterThanMostPopular() {
|
||||
|
||||
public List<Float> getHighterThanMostPopular(){
|
||||
Float mostPopular = getMostPopular();
|
||||
List<Float> higher = new ArrayList<>();
|
||||
for (Float value : countPerValue.keySet()) {
|
||||
if (value > mostPopular) {
|
||||
for(Float value: countPerValue.keySet()){
|
||||
if(value > mostPopular){
|
||||
higher.add(value);
|
||||
}
|
||||
}
|
||||
@@ -63,11 +60,12 @@ public class FloatFrequencyCounter {
|
||||
}
|
||||
|
||||
|
||||
public Float getHighest() {
|
||||
|
||||
public Float getHighest(){
|
||||
Float highest = null;
|
||||
for (Float value : countPerValue.keySet()) {
|
||||
if (highest == null || value > highest) {
|
||||
for(Float value: countPerValue.keySet()){
|
||||
if (highest == null){
|
||||
highest = value;
|
||||
} else if(value > highest){
|
||||
highest = value;
|
||||
}
|
||||
}
|
||||
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Footer {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Header {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
public enum Orientation {
|
||||
|
||||
NONE,
|
||||
LEFT,
|
||||
RIGHT
|
||||
}
|
||||
+2
-11
@@ -1,11 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
@@ -18,8 +16,6 @@ public class Page {
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
private boolean landscape;
|
||||
@@ -32,13 +28,8 @@ public class Page {
|
||||
private StringFrequencyCounter fontCounter = new StringFrequencyCounter();
|
||||
private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
|
||||
|
||||
private float pageWidth;
|
||||
private float pageHeight;
|
||||
|
||||
|
||||
public boolean isRotated() {
|
||||
|
||||
return rotation != 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+5
-14
@@ -1,6 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
@@ -8,15 +10,11 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class Paragraph implements Comparable {
|
||||
public class Paragraph {
|
||||
|
||||
private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private String headline;
|
||||
|
||||
|
||||
@@ -55,11 +53,4 @@ public class Paragraph implements Comparable {
|
||||
return textBlocks;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(Object o) {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SectionText {
|
||||
|
||||
private int sectionNumber;
|
||||
private String text;
|
||||
|
||||
private boolean isTable;
|
||||
private String headline;
|
||||
|
||||
@Builder.Default
|
||||
private List<SectionArea> sectionAreas = new ArrayList<>();
|
||||
@Builder.Default
|
||||
private Set<Image> images = new HashSet<>();
|
||||
@Builder.Default
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
@Builder.Default
|
||||
private Map<String, CellValue> tabularData = new HashMap<>();
|
||||
@Builder.Default
|
||||
private List<Integer> cellStarts = new ArrayList<>();
|
||||
|
||||
|
||||
public void setTabularData(Map<String, CellValue> tabularData) {
|
||||
|
||||
tabularData.remove(null);
|
||||
this.tabularData = tabularData;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> {
|
||||
if (block != null) {
|
||||
searchableText.addAll(block.getSequences());
|
||||
}
|
||||
});
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SimplifiedSectionText {
|
||||
|
||||
private int sectionNumber;
|
||||
private String text;
|
||||
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class SimplifiedText {
|
||||
|
||||
private int numberOfPages;
|
||||
private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
|
||||
|
||||
}
|
||||
+6
-4
@@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
public class StringFrequencyCounter {
|
||||
|
||||
@Getter
|
||||
@@ -37,11 +37,13 @@ public class StringFrequencyCounter {
|
||||
|
||||
Map.Entry<String, Integer> mostPopular = null;
|
||||
for (Map.Entry<String, Integer> entry : countPerValue.entrySet()) {
|
||||
if (mostPopular == null || entry.getValue() > mostPopular.getValue()) {
|
||||
if (mostPopular == null) {
|
||||
mostPopular = entry;
|
||||
} else if (entry.getValue() > mostPopular.getValue()) {
|
||||
mostPopular = entry;
|
||||
}
|
||||
}
|
||||
return mostPopular != null ? mostPopular.getKey() : null;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Text {
|
||||
|
||||
private int numberOfPages;
|
||||
private List<SectionText> sectionTexts = new ArrayList<>();
|
||||
|
||||
}
|
||||
+12
-183
@@ -3,189 +3,34 @@ package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@AllArgsConstructor
|
||||
@Builder
|
||||
@Data
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
@Builder.Default
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
|
||||
@JsonIgnore
|
||||
private int rotation;
|
||||
|
||||
@JsonIgnore
|
||||
private String mostPopularWordFont;
|
||||
|
||||
@JsonIgnore
|
||||
private String mostPopularWordStyle;
|
||||
|
||||
@JsonIgnore
|
||||
private float mostPopularWordFontSize;
|
||||
|
||||
@JsonIgnore
|
||||
private float mostPopularWordHeight;
|
||||
|
||||
@JsonIgnore
|
||||
private float mostPopularWordSpaceWidth;
|
||||
|
||||
@JsonIgnore
|
||||
private float highestFontSize;
|
||||
|
||||
@JsonIgnore
|
||||
private String classification;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public TextDirection getDir() {
|
||||
|
||||
return sequences.get(0).getDir();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
private float getPageHeight() {
|
||||
|
||||
return sequences.get(0).getPageHeight();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
private float getPageWidth() {
|
||||
|
||||
return sequences.get(0).getPageWidth();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the minX value in pdf coordinate system.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* @return the minX value in pdf coordinate system
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getPdfMinX() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minY;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - maxX;
|
||||
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
|
||||
return getPageWidth() - maxY;
|
||||
} else {
|
||||
return minX;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the maxX value in pdf coordinate system.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* @return the maxX value in pdf coordinate system
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getPdfMaxX() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxY;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return getPageWidth() - minX;
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageWidth() - minY;
|
||||
|
||||
} else {
|
||||
return maxX;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the minY value in pdf coordinate system.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* @return the minY value in pdf coordinate system
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getPdfMinY() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return minX;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
return maxY;
|
||||
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - maxX;
|
||||
|
||||
} else {
|
||||
return getPageHeight() - maxY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the maxY value in pdf coordinate system.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* @return the maxY value in pdf coordinate system
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getPdfMaxY() {
|
||||
|
||||
if (getDir().getDegrees() == 90) {
|
||||
return maxX;
|
||||
} else if (getDir().getDegrees() == 180) {
|
||||
|
||||
return minY;
|
||||
} else if (getDir().getDegrees() == 270) {
|
||||
return getPageHeight() - minX;
|
||||
} else {
|
||||
return getPageHeight() - minY;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
|
||||
|
||||
this.minX = minX;
|
||||
this.maxX = maxX;
|
||||
this.minY = minY;
|
||||
@@ -194,25 +39,19 @@ public class TextBlock extends AbstractTextContainer {
|
||||
this.rotation = rotation;
|
||||
}
|
||||
|
||||
|
||||
public TextBlock union(TextPositionSequence r) {
|
||||
|
||||
TextBlock union = this.copy();
|
||||
union.add(r);
|
||||
return union;
|
||||
}
|
||||
|
||||
|
||||
public TextBlock union(TextBlock r) {
|
||||
|
||||
TextBlock union = this.copy();
|
||||
union.add(r);
|
||||
return union;
|
||||
}
|
||||
|
||||
|
||||
public void add(TextBlock r) {
|
||||
|
||||
if (r.getMinX() < minX) {
|
||||
minX = r.getMinX();
|
||||
}
|
||||
@@ -228,38 +67,30 @@ public class TextBlock extends AbstractTextContainer {
|
||||
sequences.addAll(r.getSequences());
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPositionSequence r) {
|
||||
|
||||
if (r.getMinXDirAdj() < minX) {
|
||||
minX = r.getMinXDirAdj();
|
||||
if (r.getX1() < minX) {
|
||||
minX = r.getX1();
|
||||
}
|
||||
if (r.getMaxXDirAdj() > maxX) {
|
||||
maxX = r.getMaxXDirAdj();
|
||||
if (r.getX2() > maxX) {
|
||||
maxX = r.getX2();
|
||||
}
|
||||
if (r.getMinYDirAdj() < minY) {
|
||||
minY = r.getMinYDirAdj();
|
||||
if (r.getY1() < minY) {
|
||||
minY = r.getY1();
|
||||
}
|
||||
if (r.getMaxYDirAdj() > maxY) {
|
||||
maxY = r.getMaxYDirAdj();
|
||||
if (r.getY2() > maxY) {
|
||||
maxY = r.getY2();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public TextBlock copy() {
|
||||
|
||||
return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
|
||||
}
|
||||
|
||||
|
||||
public void resize(float x1, float y1, float width, float height) {
|
||||
|
||||
set(x1, y1, x1 + width, y1 + height);
|
||||
}
|
||||
|
||||
|
||||
public void set(float x1, float y1, float x2, float y2) {
|
||||
|
||||
this.minX = Math.min(x1, x2);
|
||||
this.maxX = Math.max(x1, x2);
|
||||
this.minY = Math.min(y1, y2);
|
||||
@@ -267,6 +98,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
@@ -285,10 +117,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public String getText() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
@@ -296,7 +125,7 @@ public class TextBlock extends AbstractTextContainer {
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : sequences) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
@@ -310,4 +139,4 @@ public class TextBlock extends AbstractTextContainer {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.model;
|
||||
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class UnclassifiedText {
|
||||
|
||||
private List<TextBlock> textBlocks;
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public SearchableText getSearchableText() {
|
||||
|
||||
SearchableText searchableText = new SearchableText();
|
||||
textBlocks.forEach(block -> searchableText.addAll(block.getSequences()));
|
||||
return searchableText;
|
||||
}
|
||||
|
||||
}
|
||||
+99
-131
@@ -1,40 +1,26 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import static java.util.stream.Collectors.toSet;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("all")
|
||||
public class BlockificationService {
|
||||
|
||||
static final float THRESHOLD = 1f;
|
||||
|
||||
|
||||
/**
|
||||
* This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
|
||||
* This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
|
||||
* Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
|
||||
* @param textPositions The words of a page.
|
||||
* @param horizontalRulingLines Horizontal table lines.
|
||||
* @param verticalRulingLines Vertical table lines.
|
||||
* @return Page object that contains the Textblock and text statistics.
|
||||
*/
|
||||
public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
|
||||
|
||||
List<TextPositionSequence> chunkWords = new ArrayList<>();
|
||||
@@ -43,41 +29,21 @@ public class BlockificationService {
|
||||
float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
|
||||
TextPositionSequence prev = null;
|
||||
|
||||
boolean wasSplitted = false;
|
||||
Float splitX1 = null;
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
|
||||
boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
|
||||
boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
|
||||
boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
|
||||
boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
|
||||
boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
|
||||
boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
|
||||
boolean startFromTop = word.getY1() > maxY + word.getHeight();
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
|
||||
|
||||
Orientation prevOrientation = null;
|
||||
if (!chunkBlockList1.isEmpty()) {
|
||||
prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
|
||||
}
|
||||
if (prev != null && (lineSeparation || startFromTop || word.getRotation() == 0 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), verticalRulingLines) || word.getRotation() == 0 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(maxX, minY, word
|
||||
.getX1(), word.getY1(), horizontalRulingLines) || word.getRotation() == 90 && isSplittedByRuling(minX, minY, word
|
||||
.getX1(), word.getY2(), verticalRulingLines))) {
|
||||
|
||||
TextBlock cb1 = buildTextBlock(chunkWords);
|
||||
chunkBlockList1.add(cb1);
|
||||
chunkWords = new ArrayList<>();
|
||||
|
||||
if (splitByX && !isSplitByRuling) {
|
||||
wasSplitted = true;
|
||||
cb1.setOrientation(Orientation.LEFT);
|
||||
splitX1 = word.getMinXDirAdj();
|
||||
} else if (newLineAfterSplit && !isSplitByRuling) {
|
||||
wasSplitted = false;
|
||||
cb1.setOrientation(Orientation.RIGHT);
|
||||
splitX1 = null;
|
||||
} else if (prevOrientation != null && prevOrientation.equals(Orientation.RIGHT) && (lineSeparation || !startFromTop || !splitByX || !newLineAfterSplit || !isSplitByRuling)) {
|
||||
cb1.setOrientation(Orientation.LEFT);
|
||||
}
|
||||
|
||||
minX = 1000;
|
||||
maxX = 0;
|
||||
minY = 1000;
|
||||
@@ -88,17 +54,17 @@ public class BlockificationService {
|
||||
chunkWords.add(word);
|
||||
|
||||
prev = word;
|
||||
if (word.getMinXDirAdj() < minX) {
|
||||
minX = word.getMinXDirAdj();
|
||||
if (word.getX1() < minX) {
|
||||
minX = word.getX1();
|
||||
}
|
||||
if (word.getMaxXDirAdj() > maxX) {
|
||||
maxX = word.getMaxXDirAdj();
|
||||
if (word.getX2() > maxX) {
|
||||
maxX = word.getX2();
|
||||
}
|
||||
if (word.getMinYDirAdj() < minY) {
|
||||
minY = word.getMinYDirAdj();
|
||||
if (word.getY1() < minY) {
|
||||
minY = word.getY1();
|
||||
}
|
||||
if (word.getMaxYDirAdj() > maxY) {
|
||||
maxY = word.getMaxYDirAdj();
|
||||
if (word.getY2() > maxY) {
|
||||
maxY = word.getY2();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,62 +73,10 @@ public class BlockificationService {
|
||||
chunkBlockList1.add(cb1);
|
||||
}
|
||||
|
||||
Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
|
||||
|
||||
TextBlock previousLeft = null;
|
||||
TextBlock previousRight = null;
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
|
||||
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
|
||||
previousLeft.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
|
||||
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
|
||||
previousRight.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (block.getOrientation().equals(Orientation.LEFT)) {
|
||||
previousLeft = block;
|
||||
} else if (block.getOrientation().equals(Orientation.RIGHT)) {
|
||||
previousRight = block;
|
||||
}
|
||||
}
|
||||
|
||||
itty = chunkBlockList1.iterator();
|
||||
TextBlock previous = null;
|
||||
while (itty.hasNext()) {
|
||||
TextBlock block = (TextBlock) itty.next();
|
||||
|
||||
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
|
||||
previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
|
||||
.equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
|
||||
previous.add(block);
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
|
||||
previous = block;
|
||||
}
|
||||
|
||||
return new Page(chunkBlockList1);
|
||||
}
|
||||
|
||||
|
||||
private boolean equalsWithThreshold(float f1, float f2) {
|
||||
|
||||
return Math.abs(f1 - f2) < THRESHOLD;
|
||||
}
|
||||
|
||||
|
||||
private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {
|
||||
|
||||
TextBlock textBlock = null;
|
||||
@@ -182,10 +96,12 @@ public class BlockificationService {
|
||||
styleFrequencyCounter.add(wordBlock.getFontStyle());
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation());
|
||||
textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
|
||||
.getRotation());
|
||||
} else {
|
||||
TextBlock spatialEntity = textBlock.union(wordBlock);
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
|
||||
textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
|
||||
.getHeight());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,33 +114,14 @@ public class BlockificationService {
|
||||
textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
|
||||
}
|
||||
|
||||
if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getMinYDirAdj(), 3)).collect(toSet()).size() == 1) {
|
||||
textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getMinXDirAdj));
|
||||
}
|
||||
return textBlock;
|
||||
}
|
||||
|
||||
|
||||
private boolean isSplitByRuling(float minX,
|
||||
float minY,
|
||||
float maxX,
|
||||
float maxY,
|
||||
TextPositionSequence word,
|
||||
List<Ruling> horizontalRulingLines,
|
||||
List<Ruling> verticalRulingLines) {
|
||||
|
||||
return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
|
||||
|| isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
|
||||
|| isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
|
||||
|| isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); //
|
||||
}
|
||||
|
||||
|
||||
private boolean isSplitByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines, float dir, float pageWidth, float pageHeight) {
|
||||
private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1, List<Ruling> rulingLines) {
|
||||
|
||||
for (Ruling ruling : rulingLines) {
|
||||
var line = RulingTextDirAdjustUtil.convertToDirAdj(ruling, dir, pageWidth, pageHeight);
|
||||
if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
|
||||
if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -232,10 +129,81 @@ public class BlockificationService {
|
||||
}
|
||||
|
||||
|
||||
private double round(float value, int decimalPoints) {
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
var d = Math.pow(10, decimalPoints);
|
||||
return Math.round(value * d) / d;
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
float minY = 10000;
|
||||
float maxY = -100;
|
||||
|
||||
for (Page page : pages) {
|
||||
|
||||
if (page.getTextBlocks().isEmpty() || landscape != page.isLandscape()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (AbstractTextContainer container : page.getTextBlocks()) {
|
||||
|
||||
if (container instanceof TextBlock) {
|
||||
TextBlock textBlock = (TextBlock) container;
|
||||
if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
|
||||
if (approxLineCount < 2.9f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (documentFontSizeCounter.getMostPopular() != null) {
|
||||
if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
|
||||
|
||||
if (textBlock.getMinX() < minX) {
|
||||
minX = textBlock.getMinX();
|
||||
}
|
||||
if (textBlock.getMaxX() > maxX) {
|
||||
maxX = textBlock.getMaxX();
|
||||
}
|
||||
if (textBlock.getMinY() < minY) {
|
||||
minY = textBlock.getMinY();
|
||||
}
|
||||
if (textBlock.getMaxY() > maxY) {
|
||||
maxY = textBlock.getMaxY();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (container instanceof Table) {
|
||||
Table table = (Table) container;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
if (cell == null || cell.getTextBlocks() == null) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
if (textBlock.getMinX() < minX) {
|
||||
minX = textBlock.getMinX();
|
||||
}
|
||||
if (textBlock.getMaxX() > maxX) {
|
||||
maxX = textBlock.getMaxX();
|
||||
}
|
||||
if (textBlock.getMinY() < minY) {
|
||||
minY = textBlock.getMinY();
|
||||
}
|
||||
if (textBlock.getMaxY() > maxY) {
|
||||
maxY = textBlock.getMaxY();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return new Rectangle(minY, minX, maxX - minX, maxY - minY);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-161
@@ -1,161 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
@Service
|
||||
public class BodyTextFrameService {
|
||||
|
||||
/**
|
||||
* Adjusts and sets the body text frame to a page.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
* The aspect ratio of the page is also regarded.
|
||||
*
|
||||
* @param page The page
|
||||
* @param bodyTextFrame frame that contains the main text on portrait pages
|
||||
* @param landscapeBodyTextFrame frame that contains the main text on landscape pages
|
||||
*/
|
||||
public void setBodyTextFrameAdjustedToPage(Page page, Rectangle bodyTextFrame, Rectangle landscapeBodyTextFrame) {
|
||||
|
||||
Rectangle textFrame = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
|
||||
if (page.getPageWidth() > page.getPageHeight() && page.getRotation() == 270) {
|
||||
textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), page.getPageHeight() - textFrame.getTopLeft().getX() - textFrame.getWidth()),
|
||||
textFrame.getHeight(),
|
||||
textFrame.getWidth(),
|
||||
0);
|
||||
} else if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
|
||||
textFrame = new Rectangle(new Point(textFrame.getTopLeft().getY(), textFrame.getTopLeft().getX()), textFrame.getHeight(), textFrame.getWidth(), page.getPageNumber());
|
||||
} else if (page.getRotation() == 180) {
|
||||
textFrame = new Rectangle(new Point(textFrame.getTopLeft().getX(), page.getPageHeight() - textFrame.getTopLeft().getY() - textFrame.getHeight()),
|
||||
textFrame.getWidth(),
|
||||
textFrame.getHeight(),
|
||||
0);
|
||||
}
|
||||
page.setBodyTextFrame(textFrame);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Calculates the frame that contains the main text, text outside the frame will be e.g. headers or footers.
|
||||
* Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
* The aspect ratio of the page is also regarded.
|
||||
*
|
||||
* @param pages List of all pages
|
||||
* @param documentFontSizeCounter Statistics of the document
|
||||
* @param landscape Calculate for landscape or portrait
|
||||
* @return Rectangle of the text frame
|
||||
*/
|
||||
public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
|
||||
|
||||
BodyTextFrameExpansionsRectangle expansionsRectangle = new BodyTextFrameExpansionsRectangle();
|
||||
|
||||
for (Page page : pages) {
|
||||
|
||||
if (page.getTextBlocks().isEmpty() || landscape != page.isLandscape()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (AbstractTextContainer container : page.getTextBlocks()) {
|
||||
|
||||
if (container instanceof TextBlock) {
|
||||
TextBlock textBlock = (TextBlock) container;
|
||||
if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
|
||||
if (approxLineCount < 2.9f) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
|
||||
|
||||
expandRectangle(textBlock, page, expansionsRectangle);
|
||||
}
|
||||
}
|
||||
|
||||
if (container instanceof Table) {
|
||||
Table table = (Table) container;
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
if (cell == null || cell.getTextBlocks() == null) {
|
||||
continue;
|
||||
}
|
||||
for (TextBlock textBlock : cell.getTextBlocks()) {
|
||||
expandRectangle(textBlock, page, expansionsRectangle);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Rectangle(new Point(expansionsRectangle.minX, expansionsRectangle.minY),
|
||||
expansionsRectangle.maxX - expansionsRectangle.minX,
|
||||
expansionsRectangle.maxY - expansionsRectangle.minY,
|
||||
0);
|
||||
}
|
||||
|
||||
|
||||
private void expandRectangle(TextBlock textBlock, Page page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
|
||||
|
||||
if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
|
||||
if (textBlock.getPdfMinY() < expansionsRectangle.minX) {
|
||||
expansionsRectangle.minX = textBlock.getPdfMinY();
|
||||
}
|
||||
if (textBlock.getPdfMaxY() > expansionsRectangle.maxX) {
|
||||
expansionsRectangle.maxX = textBlock.getPdfMaxY();
|
||||
}
|
||||
if (textBlock.getPdfMinX() < expansionsRectangle.minY) {
|
||||
expansionsRectangle.minY = textBlock.getPdfMinX();
|
||||
}
|
||||
if (textBlock.getPdfMaxX() > expansionsRectangle.maxY) {
|
||||
expansionsRectangle.maxY = textBlock.getPdfMaxX();
|
||||
}
|
||||
} else {
|
||||
if (textBlock.getPdfMinX() < expansionsRectangle.minX) {
|
||||
expansionsRectangle.minX = textBlock.getPdfMinX();
|
||||
}
|
||||
if (textBlock.getPdfMaxX() > expansionsRectangle.maxX) {
|
||||
expansionsRectangle.maxX = textBlock.getPdfMaxX();
|
||||
}
|
||||
if (textBlock.getPdfMinY() < expansionsRectangle.minY) {
|
||||
expansionsRectangle.minY = textBlock.getPdfMinY();
|
||||
}
|
||||
if (textBlock.getPdfMaxY() > expansionsRectangle.maxY) {
|
||||
expansionsRectangle.maxY = textBlock.getPdfMaxY();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private class BodyTextFrameExpansionsRectangle {
|
||||
|
||||
float minX = 10000;
|
||||
float maxX = -100;
|
||||
float minY = 10000;
|
||||
float maxY = -100;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+29
-54
@@ -5,78 +5,66 @@ import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ClassificationService {
|
||||
|
||||
private final BodyTextFrameService bodyTextFrameService;
|
||||
private final BlockificationService blockificationService;
|
||||
|
||||
|
||||
public void classifyDocument(Document document) {
|
||||
|
||||
Rectangle bodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
|
||||
Rectangle landscapeBodyTextFrame = bodyTextFrameService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
|
||||
Rectangle bodyTextFrame = blockificationService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), false);
|
||||
Rectangle landscapeBodyTextFrame = blockificationService.calculateBodyTextFrame(document.getPages(), document.getFontSizeCounter(), true);
|
||||
|
||||
List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
|
||||
|
||||
log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
|
||||
System.out.println(document.getFontSizeCounter().getCountPerValue());
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
bodyTextFrameService.setBodyTextFrameAdjustedToPage(page, bodyTextFrame, landscapeBodyTextFrame);
|
||||
classifyPage(page, document, headlineFontSizes);
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
page.setBodyTextFrame(btf);
|
||||
classifyPage(btf, page, document, headlineFontSizes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void classifyPage(Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
if (textBlock instanceof TextBlock) {
|
||||
classifyBlock((TextBlock) textBlock, page, document, headlineFontSizes);
|
||||
classifyBlock((TextBlock) textBlock, bodyTextFrame, page, document, headlineFontSizes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
var bodyTextFrame = page.getBodyTextFrame();
|
||||
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
textBlock.setClassification("Other");
|
||||
// TODO Figure out why this happens.
|
||||
return;
|
||||
}
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Header");
|
||||
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
|
||||
document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
|
||||
.size() == 1)) {
|
||||
} else if (page.getPageNumber() == 1
|
||||
&& (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock)
|
||||
&& PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
|
||||
.getCountPerValue()
|
||||
.containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && textBlock.getMostPopularWordStyle().equals("bold")) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
@@ -84,33 +72,20 @@ public class ClassificationService {
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
|
||||
textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
|
||||
.get(0)
|
||||
.getTextPositions()
|
||||
.get(0)
|
||||
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
}else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont().equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle().equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
}
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter().getMostPopular().equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)){
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
} else {
|
||||
textBlock.setClassification("Other");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+20
-44
@@ -1,27 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
@SuppressWarnings("all")
|
||||
public final class PositionUtils {
|
||||
public class PositionUtils {
|
||||
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
//TODO Currently this is not working for rotated pages.
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
double threshold = textBlock.getMostPopularWordHeight() * 3;
|
||||
|
||||
if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
|
||||
.getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
|
||||
.getY() + btf.getHeight()) {
|
||||
if (textBlock.getMinX() + threshold > btf.getX() &&
|
||||
textBlock.getMaxX() - threshold < btf.getX() + btf.getWidth() &&
|
||||
textBlock.getMinY() + threshold > btf.getY() &&
|
||||
textBlock.getMaxY() - threshold < btf.getY() + btf.getHeight()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@@ -30,27 +32,16 @@ public final class PositionUtils {
|
||||
}
|
||||
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
|
||||
public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, boolean rotated) {
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rotation == 90 && textBlock.getPdfMaxX() < btf.getTopLeft().getX()) {
|
||||
if (rotated && textBlock.getMinX() < btf.getX()) {
|
||||
// Its very strange, P{0,0} is on top left in this case, instead of lower left.
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 180 && textBlock.getPdfMaxY() < btf.getTopLeft().getY()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 270 && textBlock.getPdfMinX() > btf.getTopLeft().getX() + btf.getWidth()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 0 && textBlock.getPdfMinY() > btf.getTopLeft().getY() + btf.getHeight()) {
|
||||
} else if (!rotated && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@@ -58,27 +49,16 @@ public final class PositionUtils {
|
||||
|
||||
}
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
|
||||
|
||||
public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
//TODO Currently this is not working for rotated pages.
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rotation == 90 && textBlock.getPdfMinX() > btf.getTopLeft().getX() + btf.getWidth()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 180 && textBlock.getPdfMinY() > btf.getTopLeft().getY() + btf.getHeight()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 270 && textBlock.getPdfMaxX() < btf.getTopLeft().getX()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (rotation == 0 && textBlock.getPdfMaxY() < btf.getTopLeft().getY()) {
|
||||
if (textBlock.getMaxY() < btf.getY()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@@ -86,8 +66,7 @@ public final class PositionUtils {
|
||||
|
||||
}
|
||||
|
||||
// TODO This currently uses pdf coord system. In the futher this should use java coord system.
|
||||
// Note: DirAdj (TextDirection Adjusted) can not be user for this.
|
||||
|
||||
public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
//TODO Currently this is not working for rotated pages.
|
||||
@@ -96,7 +75,7 @@ public final class PositionUtils {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (textBlock.getMinY() < btf.getTopLeft().getY()) {
|
||||
if (textBlock.getMinY() < btf.getY()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@@ -106,14 +85,11 @@ public final class PositionUtils {
|
||||
|
||||
|
||||
public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
|
||||
|
||||
return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
|
||||
}
|
||||
|
||||
|
||||
public Float getApproxLineCount(TextBlock textBlock) {
|
||||
|
||||
return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-64
@@ -1,64 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.utils;
|
||||
|
||||
import java.awt.geom.Line2D;
|
||||
import java.awt.geom.Point2D;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public final class RulingTextDirAdjustUtil {
|
||||
|
||||
/**
|
||||
* Converts a ruling (line of a table) the same way TextPositions are converted in PDFBox.
|
||||
* This will get the y position of the text, adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
*
|
||||
* See org.apache.pdfbox.text.TextPosition
|
||||
*/
|
||||
public Line2D.Float convertToDirAdj(Ruling ruling, float dir, float pageWidth, float pageHeight) {
|
||||
|
||||
return new Line2D.Float(convertPoint(ruling.x1, ruling.y1, dir, pageWidth, pageHeight), convertPoint(ruling.x2, ruling.y2, dir, pageWidth, pageHeight));
|
||||
}
|
||||
|
||||
|
||||
private Point2D convertPoint(float x, float y, float dir, float pageWidth, float pageHeight) {
|
||||
|
||||
var xAdj = getXRot(x, y, dir, pageWidth, pageHeight);
|
||||
var yLowerLeftRot = getYLowerLeftRot(x, y, dir, pageWidth, pageHeight);
|
||||
var yAdj = dir == 0 || dir == 180 ? pageHeight - yLowerLeftRot : pageWidth - yLowerLeftRot;
|
||||
return new Point2D.Float(xAdj, yAdj);
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("SuspiciousNameCombination")
|
||||
private float getXRot(float x, float y, float dir, float pageWidth, float pageHeight) {
|
||||
|
||||
if (dir == 0) {
|
||||
return x;
|
||||
} else if (dir == 90) {
|
||||
return y;
|
||||
} else if (dir == 180) {
|
||||
return pageWidth - x;
|
||||
} else if (dir == 270) {
|
||||
return pageHeight - y;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
private float getYLowerLeftRot(float x, float y, float dir, float pageWidth, float pageHeight) {
|
||||
|
||||
if (dir == 0) {
|
||||
return y;
|
||||
} else if (dir == 90) {
|
||||
return pageWidth - x;
|
||||
} else if (dir == 180) {
|
||||
return pageHeight - y;
|
||||
} else if (dir == 270) {
|
||||
return x;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
+4
-4
@@ -2,9 +2,9 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = "DictionaryResource", url = "${persistence-service.url}")
|
||||
@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface DictionaryClient extends DictionaryResource {
|
||||
|
||||
}
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.FileStatusProcessingUpdateResource;
|
||||
|
||||
@FeignClient(name = "FileStatusProcessingUpdateResource", url = "${persistence-service.url}")
|
||||
public interface FileStatusProcessingUpdateClient extends FileStatusProcessingUpdateResource {
|
||||
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.LegalBasisMappingResource;
|
||||
|
||||
@FeignClient(name = "LegalBasisMappingResource", url = "${persistence-service.url}")
|
||||
public interface LegalBasisClient extends LegalBasisMappingResource {
|
||||
|
||||
}
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
public class MockMultipartFile implements MultipartFile {
|
||||
|
||||
private final String name;
|
||||
private final String originalFilename;
|
||||
@Nullable
|
||||
private final String contentType;
|
||||
private final byte[] content;
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable byte[] content) {
|
||||
|
||||
this(name, "", null, content);
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, InputStream contentStream) throws IOException {
|
||||
|
||||
this(name, "", null, FileCopyUtils.copyToByteArray(contentStream));
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType, @Nullable byte[] content) {
|
||||
|
||||
Assert.hasLength(name, "Name must not be empty");
|
||||
this.name = name;
|
||||
this.originalFilename = originalFilename != null ? originalFilename : "";
|
||||
this.contentType = contentType;
|
||||
this.content = content != null ? content : new byte[0];
|
||||
}
|
||||
|
||||
|
||||
public MockMultipartFile(String name, @Nullable String originalFilename, @Nullable String contentType, InputStream contentStream) throws IOException {
|
||||
|
||||
this(name, originalFilename, contentType, FileCopyUtils.copyToByteArray(contentStream));
|
||||
}
|
||||
|
||||
|
||||
public String getName() {
|
||||
|
||||
return this.name;
|
||||
}
|
||||
|
||||
|
||||
@NonNull
|
||||
public String getOriginalFilename() {
|
||||
|
||||
return this.originalFilename;
|
||||
}
|
||||
|
||||
|
||||
@Nullable
|
||||
public String getContentType() {
|
||||
|
||||
return this.contentType;
|
||||
}
|
||||
|
||||
|
||||
public boolean isEmpty() {
|
||||
|
||||
return this.content.length == 0;
|
||||
}
|
||||
|
||||
|
||||
public long getSize() {
|
||||
|
||||
return this.content.length;
|
||||
}
|
||||
|
||||
|
||||
public byte[] getBytes() {
|
||||
|
||||
return this.content;
|
||||
}
|
||||
|
||||
|
||||
public InputStream getInputStream() {
|
||||
|
||||
return new ByteArrayInputStream(this.content);
|
||||
}
|
||||
|
||||
|
||||
public void transferTo(File dest) throws IOException, IllegalStateException {
|
||||
|
||||
FileCopyUtils.copy(this.content, dest);
|
||||
}
|
||||
|
||||
}
|
||||
+3
-4
@@ -2,9 +2,8 @@ package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.RulesResource;
|
||||
import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
|
||||
|
||||
@FeignClient(name = "RulesResource", url = "${persistence-service.url}")
|
||||
@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
|
||||
public interface RulesClient extends RulesResource {
|
||||
|
||||
}
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.internal.resources.TenantsResource;
|
||||
|
||||
@FeignClient(name = "TenantsResource", url = "${persistence-service.url}")
|
||||
public interface TenantsClient extends TenantsResource {
|
||||
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class EntityRecogintionEntity {
|
||||
|
||||
private String value;
|
||||
private int startOffset;
|
||||
private int endOffset;
|
||||
private String type;
|
||||
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class NerEntities {
|
||||
|
||||
private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
|
||||
|
||||
}
|
||||
+6
-20
@@ -1,18 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import com.iqser.red.commons.spring.ErrorMessage;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.web.bind.annotation.ExceptionHandler;
|
||||
import org.springframework.web.bind.annotation.ResponseBody;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestControllerAdvice;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestControllerAdvice
|
||||
@@ -20,12 +19,10 @@ public class ControllerAdvice {
|
||||
|
||||
/* error handling */
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR)
|
||||
@ExceptionHandler(value = NullPointerException.class)
|
||||
public ErrorMessage handleContentNotFoundException(NullPointerException e) {
|
||||
|
||||
if (e != null) {
|
||||
log.error(e.getMessage(), e);
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
@@ -34,22 +31,11 @@ public class ControllerAdvice {
|
||||
return new ErrorMessage(OffsetDateTime.now(), "Nullpointer exception");
|
||||
}
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.BAD_REQUEST)
|
||||
@ExceptionHandler(value = RulesValidationException.class)
|
||||
public ErrorMessage handleRulesValidationException(RulesValidationException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
|
||||
@ResponseBody
|
||||
@ResponseStatus(value = HttpStatus.NOT_FOUND)
|
||||
@ExceptionHandler(value = NotFoundException.class)
|
||||
public ErrorMessage handleFileNotFoundException(NotFoundException e) {
|
||||
|
||||
return new ErrorMessage(OffsetDateTime.now(), e.getMessage());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+140
-8
@@ -1,31 +1,163 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.EntityRedactionService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.AnnotationHighlightService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfFlattenService;
|
||||
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfVisualisationService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final PdfVisualisationService pdfVisualisationService;
|
||||
private final PdfSegmentationService pdfSegmentationService;
|
||||
private final AnnotationHighlightService annotationHighlightService;
|
||||
private final EntityRedactionService entityRedactionService;
|
||||
private final PdfFlattenService pdfFlattenService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
|
||||
|
||||
@Override
|
||||
public void testRules(@RequestBody String rules) {
|
||||
public RedactionResult redact(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try {
|
||||
droolsExecutionService.testRules(rules);
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
entityRedactionService.processDocument(classifiedDoc, redactionRequest.getManualRedactions());
|
||||
annotationHighlightService.highlight(pdDocument, classifiedDoc, redactionRequest.isFlatRedaction(), redactionRequest
|
||||
.getManualRedactions());
|
||||
|
||||
if (redactionRequest.isFlatRedaction()) {
|
||||
PDDocument flatDocument = pdfFlattenService.flattenPDF(pdDocument);
|
||||
return convert(flatDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
}
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages()
|
||||
.size(), classifiedDoc.getRedactionLogEntities(), classifiedDoc.getSectionGrid(), classifiedDoc.getDictionaryVersion(), classifiedDoc.getRulesVersion());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult classify(@RequestBody RedactionRequest pdfSegmentationRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(pdfSegmentationRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult sections(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
|
||||
|
||||
return convert(pdDocument, classifiedDoc.getPages().size());
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public RedactionResult htmlTables(@RequestBody RedactionRequest redactionRequest) {
|
||||
|
||||
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Page page : classifiedDoc.getPages()) {
|
||||
for (AbstractTextContainer textContainer : page.getTextBlocks()) {
|
||||
if (textContainer instanceof Table) {
|
||||
Table table = (Table) textContainer;
|
||||
sb.append(table.getTextAsHtml()).append("<br />").append("<br />");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RedactionResult.builder().document(sb.toString().getBytes()).build();
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void updateRules(@RequestBody String rules) {
|
||||
|
||||
droolsExecutionService.updateRules(rules);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
|
||||
|
||||
return convert(document, numberOfPages, null, null, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
private RedactionResult convert(PDDocument document, int numberOfPages,
|
||||
List<RedactionLogEntry> redactionLogEntities,
|
||||
SectionGrid sectionGrid, long dictionaryVersion, long rulesVersion) throws IOException {
|
||||
|
||||
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
|
||||
document.save(byteArrayOutputStream);
|
||||
return RedactionResult.builder()
|
||||
.document(byteArrayOutputStream.toByteArray())
|
||||
.numberOfPages(numberOfPages)
|
||||
.redactionLog(new RedactionLog(redactionLogEntities,dictionaryVersion, rulesVersion))
|
||||
.sectionGrid(sectionGrid)
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
|
||||
import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.rulebuilder.RuleBuilderModelService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
@RestController
|
||||
@RequiredArgsConstructor
|
||||
public class RuleBuilderController implements RuleBuilderResource {
|
||||
|
||||
private final RuleBuilderModelService ruleBuilderModelService;
|
||||
|
||||
|
||||
@Override
|
||||
public RuleBuilderModel getRuleBuilderModel() {
|
||||
|
||||
return ruleBuilderModelService.getRuleBuilderModel();
|
||||
}
|
||||
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.exception;
|
||||
|
||||
public class NotFoundException extends RuntimeException {
|
||||
|
||||
public NotFoundException(String message) {
|
||||
|
||||
super(message);
|
||||
}
|
||||
|
||||
}
|
||||
-3
@@ -3,13 +3,10 @@ package com.iqser.red.service.redaction.v1.server.exception;
|
||||
public class RedactionException extends RuntimeException {
|
||||
|
||||
public RedactionException(Throwable cause) {
|
||||
|
||||
super("Could not parse document", cause);
|
||||
}
|
||||
|
||||
|
||||
public RedactionException() {
|
||||
|
||||
super("Could not parse document");
|
||||
}
|
||||
|
||||
|
||||
-1
@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.exception;
|
||||
public class RulesValidationException extends RuntimeException {
|
||||
|
||||
public RulesValidationException(String message, Throwable t) {
|
||||
|
||||
super(message, t);
|
||||
}
|
||||
|
||||
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.scheduling.annotation.AsyncConfigurerSupport;
|
||||
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
|
||||
|
||||
@Configuration
|
||||
public class AsyncConfig extends AsyncConfigurerSupport {
|
||||
|
||||
@Override
|
||||
public Executor getAsyncExecutor() {
|
||||
|
||||
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
|
||||
|
||||
executor.setCorePoolSize(7);
|
||||
executor.setMaxPoolSize(42);
|
||||
executor.setQueueCapacity(11);
|
||||
executor.setThreadNamePrefix("TenantAwareTaskExecutor-");
|
||||
executor.setTaskDecorator(new TenantAwareTaskDecorator());
|
||||
executor.initialize();
|
||||
|
||||
return executor;
|
||||
}
|
||||
|
||||
}
|
||||
-105
@@ -1,105 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.spec.KeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.SecretKeyFactory;
|
||||
import javax.crypto.spec.GCMParameterSpec;
|
||||
import javax.crypto.spec.PBEKeySpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Service
|
||||
public class EncryptionDecryptionService {
|
||||
|
||||
@Value("${redaction-service.crypto.key:redaction}")
|
||||
private String key;
|
||||
|
||||
private SecretKey secretKey;
|
||||
private byte[] iv;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@PostConstruct
|
||||
protected void postConstruct() {
|
||||
|
||||
SecureRandom secureRandom = new SecureRandom();
|
||||
iv = new byte[12];
|
||||
secureRandom.nextBytes(iv);
|
||||
secretKey = generateSecretKey(key, iv);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public String encrypt(String strToEncrypt) {
|
||||
|
||||
return Base64.getEncoder().encodeToString(encrypt(strToEncrypt.getBytes()));
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public String decrypt(String strToDecrypt) {
|
||||
|
||||
byte[] bytes = Base64.getDecoder().decode(strToDecrypt);
|
||||
return new String(decrypt(bytes), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public byte[] encrypt(byte[] data) {
|
||||
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey, parameterSpec);
|
||||
byte[] encryptedData = cipher.doFinal(data);
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(4 + iv.length + encryptedData.length);
|
||||
byteBuffer.putInt(iv.length);
|
||||
byteBuffer.put(iv);
|
||||
byteBuffer.put(encryptedData);
|
||||
return byteBuffer.array();
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public byte[] decrypt(byte[] encryptedData) {
|
||||
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(encryptedData);
|
||||
int noonceSize = byteBuffer.getInt();
|
||||
if (noonceSize < 12 || noonceSize >= 16) {
|
||||
throw new IllegalArgumentException("Nonce size is incorrect. Make sure that the incoming data is an AES encrypted file.");
|
||||
}
|
||||
byte[] iv = new byte[noonceSize];
|
||||
byteBuffer.get(iv);
|
||||
|
||||
SecretKey secretKey = generateSecretKey(key, iv);
|
||||
|
||||
byte[] cipherBytes = new byte[byteBuffer.remaining()];
|
||||
byteBuffer.get(cipherBytes);
|
||||
|
||||
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
|
||||
GCMParameterSpec parameterSpec = new GCMParameterSpec(128, iv);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey, parameterSpec);
|
||||
return cipher.doFinal(cipherBytes);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public SecretKey generateSecretKey(String password, byte[] iv) {
|
||||
|
||||
KeySpec spec = new PBEKeySpec(password.toCharArray(), iv, 65536, 128); // AES-128
|
||||
SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
|
||||
byte[] key = secretKeyFactory.generateSecret(spec).getEncoded();
|
||||
return new SecretKeySpec(key, "AES");
|
||||
}
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import feign.RequestInterceptor;
|
||||
import feign.RequestTemplate;
|
||||
|
||||
@Component
|
||||
public class ForwardTenantInterceptor implements RequestInterceptor {
|
||||
|
||||
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
|
||||
|
||||
@Override
|
||||
public void apply(RequestTemplate template) {
|
||||
template.header(TENANT_HEADER_NAME, TenantContext.getTenantId());
|
||||
}
|
||||
}
|
||||
-49
@@ -1,49 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.multitenancy.ForwardTenantInterceptor.TENANT_HEADER_NAME;
|
||||
|
||||
import org.springframework.amqp.rabbit.config.AbstractRabbitListenerContainerFactory;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.config.BeanPostProcessor;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
public class MultiTenancyMessagingConfiguration {
|
||||
|
||||
@Bean
|
||||
public static BeanPostProcessor multitenancyBeanPostProcessor() {
|
||||
|
||||
return new BeanPostProcessor() {
|
||||
|
||||
@Override
|
||||
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
|
||||
|
||||
if (bean instanceof RabbitTemplate) {
|
||||
|
||||
((RabbitTemplate) bean).setBeforePublishPostProcessors(m -> {
|
||||
m.getMessageProperties().setHeader(TENANT_HEADER_NAME, TenantContext.getTenantId());
|
||||
return m;
|
||||
});
|
||||
|
||||
} else if (bean instanceof AbstractRabbitListenerContainerFactory) {
|
||||
|
||||
((AbstractRabbitListenerContainerFactory<?>) bean).setAfterReceivePostProcessors(m -> {
|
||||
String tenant = m.getMessageProperties().getHeader(TENANT_HEADER_NAME);
|
||||
|
||||
if (tenant != null) {
|
||||
TenantContext.setTenantId(tenant);
|
||||
} else {
|
||||
throw new RuntimeException("No Tenant is set queue message");
|
||||
}
|
||||
return m;
|
||||
});
|
||||
}
|
||||
return bean;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
|
||||
|
||||
import com.iqser.red.commons.spring.DefaultWebMvcConfiguration;
|
||||
|
||||
@Configuration
|
||||
public class MultiTenancyWebConfiguration extends DefaultWebMvcConfiguration {
|
||||
|
||||
private final TenantInterceptor tenantInterceptor;
|
||||
|
||||
|
||||
@Autowired
|
||||
public MultiTenancyWebConfiguration(TenantInterceptor tenantInterceptor) {
|
||||
|
||||
this.tenantInterceptor = tenantInterceptor;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void addInterceptors(InterceptorRegistry registry) {
|
||||
|
||||
registry.addWebRequestInterceptor(tenantInterceptor);
|
||||
}
|
||||
|
||||
}
|
||||
-45
@@ -1,45 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.TenantsClient;
|
||||
import com.iqser.red.storage.commons.model.AzureStorageConnection;
|
||||
import com.iqser.red.storage.commons.model.S3StorageConnection;
|
||||
import com.iqser.red.storage.commons.service.StorageConnectionProvider;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class StorageConnectionProviderImpl implements StorageConnectionProvider {
|
||||
|
||||
private final TenantsClient tenantsClient;
|
||||
private final EncryptionDecryptionService encryptionDecryptionService;
|
||||
|
||||
|
||||
@Override
|
||||
public AzureStorageConnection getAzureStorageConnection(String tenantId) {
|
||||
|
||||
var tenant = tenantsClient.getTenant(tenantId);
|
||||
return AzureStorageConnection.builder()
|
||||
.connectionString(encryptionDecryptionService.decrypt(tenant.getAzureStorageConnection().getConnectionString()))
|
||||
.containerName(tenant.getAzureStorageConnection().getContainerName())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public S3StorageConnection getS3StorageConnection(String tenantId) {
|
||||
|
||||
var tenant = tenantsClient.getTenant(tenantId);
|
||||
return S3StorageConnection.builder()
|
||||
.key(tenant.getS3StorageConnection().getKey())
|
||||
.secret(encryptionDecryptionService.decrypt(tenant.getS3StorageConnection().getSecret()))
|
||||
.signerType(tenant.getS3StorageConnection().getSignerType())
|
||||
.bucketName(tenant.getS3StorageConnection().getBucketName())
|
||||
.region(tenant.getS3StorageConnection().getRegion())
|
||||
.endpoint(tenant.getS3StorageConnection().getEndpoint())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.core.task.TaskDecorator;
|
||||
import org.springframework.lang.NonNull;
|
||||
|
||||
public class TenantAwareTaskDecorator implements TaskDecorator {
|
||||
|
||||
@Override
|
||||
@NonNull
|
||||
public Runnable decorate(@NonNull Runnable runnable) {
|
||||
|
||||
String tenantId = TenantContext.getTenantId();
|
||||
return () -> {
|
||||
try {
|
||||
TenantContext.setTenantId(tenantId);
|
||||
runnable.run();
|
||||
} finally {
|
||||
TenantContext.setTenantId(null);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public final class TenantContext {
|
||||
|
||||
private static InheritableThreadLocal<String> currentTenant = new InheritableThreadLocal<>();
|
||||
|
||||
|
||||
public static void setTenantId(String tenantId) {
|
||||
|
||||
log.debug("Setting tenantId to " + tenantId);
|
||||
currentTenant.set(tenantId);
|
||||
}
|
||||
|
||||
|
||||
public static String getTenantId() {
|
||||
|
||||
return currentTenant.get();
|
||||
}
|
||||
|
||||
|
||||
public static void clear() {
|
||||
|
||||
currentTenant.remove();
|
||||
}
|
||||
|
||||
}
|
||||
-35
@@ -1,35 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.multitenancy;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.ui.ModelMap;
|
||||
import org.springframework.web.context.request.WebRequest;
|
||||
import org.springframework.web.context.request.WebRequestInterceptor;
|
||||
|
||||
@Component
|
||||
public class TenantInterceptor implements WebRequestInterceptor {
|
||||
|
||||
public static final String TENANT_HEADER_NAME = "X-TENANT-ID";
|
||||
|
||||
|
||||
@Override
|
||||
public void preHandle(WebRequest request) {
|
||||
|
||||
if (request.getHeader(TENANT_HEADER_NAME) != null) {
|
||||
TenantContext.setTenantId(request.getHeader(TENANT_HEADER_NAME));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void postHandle(WebRequest request, ModelMap model) {
|
||||
|
||||
TenantContext.clear();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void afterCompletion(WebRequest request, Exception ex) {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-386
@@ -1,386 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.WeakHashMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.fontbox.ttf.TrueTypeFont;
|
||||
import org.apache.fontbox.util.BoundingBox;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFStreamEngine;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType3Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.apache.pdfbox.util.Vector;
|
||||
import org.apache.pdfbox.contentstream.operator.DrawObject;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Concatenate;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Restore;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Save;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetGraphicsStateParameters;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetMatrix;
|
||||
import org.apache.pdfbox.contentstream.operator.text.BeginText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.EndText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveTextSetLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.NextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetCharSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRenderingMode;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRise;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetWordSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowText;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
|
||||
/**
|
||||
* LEGACY text calculations which are known to be incorrect but are depended on by PDFTextStripper.
|
||||
* <p>
|
||||
* This class exists only so that we don't break the code of users who have their own subclasses of
|
||||
* PDFTextStripper. It replaces the mostly empty implementation of showGlyph() in PDFStreamEngine
|
||||
* with a heuristic implementation which is backwards compatible.
|
||||
* <p>
|
||||
* DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
* THIS CODE IS DELIBERATELY INCORRECT, USE PDFStreamEngine INSTEAD.
|
||||
*/
|
||||
@SuppressWarnings({"PMD", "checkstyle:all"})
|
||||
class LegacyPDFStreamEngine extends PDFStreamEngine {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(LegacyPDFStreamEngine.class);
|
||||
|
||||
private int pageRotation;
|
||||
private PDRectangle pageSize;
|
||||
private Matrix translateMatrix;
|
||||
private final GlyphList glyphList;
|
||||
private final Map<COSDictionary, Float> fontHeightMap = new WeakHashMap<COSDictionary, Float>();
|
||||
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
LegacyPDFStreamEngine() throws IOException {
|
||||
|
||||
addOperator(new BeginText());
|
||||
addOperator(new Concatenate());
|
||||
addOperator(new DrawObject()); // special text version
|
||||
addOperator(new EndText());
|
||||
addOperator(new SetGraphicsStateParameters());
|
||||
addOperator(new Save());
|
||||
addOperator(new Restore());
|
||||
addOperator(new NextLine());
|
||||
addOperator(new SetCharSpacing());
|
||||
addOperator(new MoveText());
|
||||
addOperator(new MoveTextSetLeading());
|
||||
addOperator(new SetFontAndSize());
|
||||
addOperator(new ShowText());
|
||||
addOperator(new ShowTextAdjusted());
|
||||
addOperator(new SetTextLeading());
|
||||
addOperator(new SetMatrix());
|
||||
addOperator(new SetTextRenderingMode());
|
||||
addOperator(new SetTextRise());
|
||||
addOperator(new SetWordSpacing());
|
||||
addOperator(new SetTextHorizontalScaling());
|
||||
addOperator(new ShowTextLine());
|
||||
addOperator(new ShowTextLineAndSpace());
|
||||
|
||||
// load additional glyph list for Unicode mapping
|
||||
String path = "/org/apache/pdfbox/resources/glyphlist/additional.txt";
|
||||
InputStream input = GlyphList.class.getResourceAsStream(path);
|
||||
glyphList = new GlyphList(GlyphList.getAdobeGlyphList(), input);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This will initialize and process the contents of the stream.
|
||||
*
|
||||
* @param page the page to process
|
||||
* @throws java.io.IOException if there is an error accessing the stream.
|
||||
*/
|
||||
@Override
|
||||
public void processPage(PDPage page) throws IOException {
|
||||
|
||||
this.pageRotation = page.getRotation();
|
||||
this.pageSize = page.getCropBox();
|
||||
|
||||
if (pageSize.getLowerLeftX() == 0 && pageSize.getLowerLeftY() == 0) {
|
||||
translateMatrix = null;
|
||||
} else {
|
||||
// translation matrix for cropbox
|
||||
translateMatrix = Matrix.getTranslateInstance(-pageSize.getLowerLeftX(), -pageSize.getLowerLeftY());
|
||||
}
|
||||
super.processPage(page);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Called when a glyph is to be processed. The heuristic calculations here were originally
|
||||
* written by Ben Litchfield for PDFStreamEngine.
|
||||
*/
|
||||
@Override
|
||||
protected void showGlyph(Matrix textRenderingMatrix, PDFont font, int code, String unicode, Vector displacement) throws IOException {
|
||||
//
|
||||
// legacy calculations which were previously in PDFStreamEngine
|
||||
//
|
||||
// DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
// THIS CODE IS DELIBERATELY INCORRECT
|
||||
//
|
||||
|
||||
PDGraphicsState state = getGraphicsState();
|
||||
Matrix ctm = state.getCurrentTransformationMatrix();
|
||||
float fontSize = state.getTextState().getFontSize();
|
||||
float horizontalScaling = state.getTextState().getHorizontalScaling() / 100f;
|
||||
Matrix textMatrix = getTextMatrix();
|
||||
|
||||
float displacementX = displacement.getX();
|
||||
// the sorting algorithm is based on the width of the character. As the displacement
|
||||
// for vertical characters doesn't provide any suitable value for it, we have to
|
||||
// calculate our own
|
||||
if (font.isVertical()) {
|
||||
displacementX = font.getWidth(code) / 1000;
|
||||
// there may be an additional scaling factor for true type fonts
|
||||
TrueTypeFont ttf = null;
|
||||
if (font instanceof PDTrueTypeFont) {
|
||||
ttf = ((PDTrueTypeFont) font).getTrueTypeFont();
|
||||
} else if (font instanceof PDType0Font) {
|
||||
PDCIDFont cidFont = ((PDType0Font) font).getDescendantFont();
|
||||
if (cidFont instanceof PDCIDFontType2) {
|
||||
ttf = ((PDCIDFontType2) cidFont).getTrueTypeFont();
|
||||
}
|
||||
}
|
||||
if (ttf != null && ttf.getUnitsPerEm() != 1000) {
|
||||
displacementX *= 1000f / ttf.getUnitsPerEm();
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// legacy calculations which were previously in PDFStreamEngine
|
||||
//
|
||||
// DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
// THIS CODE IS DELIBERATELY INCORRECT
|
||||
//
|
||||
|
||||
// (modified) combined displacement, this is calculated *without* taking the character
|
||||
// spacing and word spacing into account, due to legacy code in TextStripper
|
||||
float tx = displacementX * fontSize * horizontalScaling;
|
||||
float ty = displacement.getY() * fontSize;
|
||||
|
||||
// (modified) combined displacement matrix
|
||||
Matrix td = Matrix.getTranslateInstance(tx, ty);
|
||||
|
||||
// (modified) text rendering matrix
|
||||
Matrix nextTextRenderingMatrix = td.multiply(textMatrix).multiply(ctm); // text space -> device space
|
||||
float nextX = nextTextRenderingMatrix.getTranslateX();
|
||||
float nextY = nextTextRenderingMatrix.getTranslateY();
|
||||
|
||||
// (modified) width and height calculations
|
||||
float dxDisplay = nextX - textRenderingMatrix.getTranslateX();
|
||||
Float fontHeight = fontHeightMap.get(font.getCOSObject());
|
||||
if (fontHeight == null) {
|
||||
fontHeight = computeFontHeight(font);
|
||||
fontHeightMap.put(font.getCOSObject(), fontHeight);
|
||||
}
|
||||
float dyDisplay = fontHeight * textRenderingMatrix.getScalingFactorY();
|
||||
|
||||
//
|
||||
// start of the original method
|
||||
//
|
||||
|
||||
// Note on variable names. There are three different units being used in this code.
|
||||
// Character sizes are given in glyph units, text locations are initially given in text
|
||||
// units, and we want to save the data in display units. The variable names should end with
|
||||
// Text or Disp to represent if the values are in text or disp units (no glyph units are
|
||||
// saved).
|
||||
|
||||
float glyphSpaceToTextSpaceFactor = 1 / 1000f;
|
||||
if (font instanceof PDType3Font) {
|
||||
glyphSpaceToTextSpaceFactor = font.getFontMatrix().getScaleX();
|
||||
}
|
||||
|
||||
float spaceWidthText = 0;
|
||||
try {
|
||||
// to avoid crash as described in PDFBOX-614, see what the space displacement should be
|
||||
spaceWidthText = font.getSpaceWidth() * glyphSpaceToTextSpaceFactor;
|
||||
} catch (Throwable exception) {
|
||||
LOG.warn(exception, exception);
|
||||
}
|
||||
|
||||
if (spaceWidthText == 0) {
|
||||
spaceWidthText = font.getAverageFontWidth() * glyphSpaceToTextSpaceFactor;
|
||||
// the average space width appears to be higher than necessary so make it smaller
|
||||
spaceWidthText *= .80f;
|
||||
}
|
||||
if (spaceWidthText == 0) {
|
||||
spaceWidthText = 1.0f; // if could not find font, use a generic value
|
||||
}
|
||||
|
||||
// the space width has to be transformed into display units
|
||||
float spaceWidthDisplay = spaceWidthText * textRenderingMatrix.getScalingFactorX();
|
||||
|
||||
// use our additional glyph list for Unicode mapping
|
||||
String unicodeMapping = font.toUnicode(code, glyphList);
|
||||
|
||||
// when there is no Unicode mapping available, Acrobat simply coerces the character code
|
||||
// into Unicode, so we do the same. Subclasses of PDFStreamEngine don't necessarily want
|
||||
// this, which is why we leave it until this point in PDFTextStreamEngine.
|
||||
if (unicodeMapping == null) {
|
||||
if (font instanceof PDSimpleFont) {
|
||||
char c = (char) code;
|
||||
unicodeMapping = new String(new char[]{c});
|
||||
} else {
|
||||
// Acrobat doesn't seem to coerce composite font's character codes, instead it
|
||||
// skips them. See the "allah2.pdf" TestTextStripper file.
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// adjust for cropbox if needed
|
||||
Matrix translatedTextRenderingMatrix;
|
||||
if (translateMatrix == null) {
|
||||
translatedTextRenderingMatrix = textRenderingMatrix;
|
||||
} else {
|
||||
translatedTextRenderingMatrix = Matrix.concatenate(translateMatrix, textRenderingMatrix);
|
||||
nextX -= pageSize.getLowerLeftX();
|
||||
nextY -= pageSize.getLowerLeftY();
|
||||
}
|
||||
|
||||
// This is a hack for unicode letter with 2 chars e.g. RA see unicodeProblem.pdf
|
||||
if (unicodeMapping.length() == 2) {
|
||||
processTextPosition(new TextPosition(pageRotation,
|
||||
pageSize.getWidth(),
|
||||
pageSize.getHeight(),
|
||||
translatedTextRenderingMatrix,
|
||||
nextX,
|
||||
nextY,
|
||||
Math.abs(dyDisplay),
|
||||
dxDisplay,
|
||||
Math.abs(spaceWidthDisplay),
|
||||
Character.toString(unicodeMapping.charAt(0)),
|
||||
new int[]{code},
|
||||
font,
|
||||
fontSize,
|
||||
(int) (fontSize * textMatrix.getScalingFactorX())));
|
||||
processTextPosition(new TextPosition(pageRotation,
|
||||
pageSize.getWidth(),
|
||||
pageSize.getHeight(),
|
||||
translatedTextRenderingMatrix,
|
||||
nextX,
|
||||
nextY,
|
||||
Math.abs(dyDisplay),
|
||||
dxDisplay,
|
||||
Math.abs(spaceWidthDisplay),
|
||||
Character.toString(unicodeMapping.charAt(1)),
|
||||
new int[]{code},
|
||||
font,
|
||||
fontSize,
|
||||
(int) (fontSize * textMatrix.getScalingFactorX())));
|
||||
} else {
|
||||
|
||||
processTextPosition(new TextPosition(pageRotation,
|
||||
pageSize.getWidth(),
|
||||
pageSize.getHeight(),
|
||||
translatedTextRenderingMatrix,
|
||||
nextX,
|
||||
nextY,
|
||||
Math.abs(dyDisplay),
|
||||
dxDisplay,
|
||||
Math.abs(spaceWidthDisplay),
|
||||
unicodeMapping,
|
||||
new int[]{code},
|
||||
font,
|
||||
fontSize,
|
||||
(int) (fontSize * textMatrix.getScalingFactorX())));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Compute the font height. Override this if you want to use own calculations.
|
||||
*
|
||||
* @param font the font.
|
||||
* @return the font height.
|
||||
* @throws IOException if there is an error while getting the font bounding box.
|
||||
*/
|
||||
protected float computeFontHeight(PDFont font) throws IOException {
|
||||
|
||||
BoundingBox bbox = font.getBoundingBox();
|
||||
if (bbox.getLowerLeftY() < Short.MIN_VALUE) {
|
||||
// PDFBOX-2158 and PDFBOX-3130
|
||||
// files by Salmat eSolutions / ClibPDF Library
|
||||
bbox.setLowerLeftY(-(bbox.getLowerLeftY() + 65536));
|
||||
}
|
||||
// 1/2 the bbox is used as the height todo: why?
|
||||
float glyphHeight = bbox.getHeight() / 2;
|
||||
|
||||
// sometimes the bbox has very high values, but CapHeight is OK
|
||||
PDFontDescriptor fontDescriptor = font.getFontDescriptor();
|
||||
if (fontDescriptor != null) {
|
||||
float capHeight = fontDescriptor.getCapHeight();
|
||||
if (Float.compare(capHeight, 0) != 0 && (capHeight < glyphHeight || Float.compare(glyphHeight, 0) == 0)) {
|
||||
glyphHeight = capHeight;
|
||||
}
|
||||
// PDFBOX-3464, PDFBOX-4480, PDFBOX-4553:
|
||||
// sometimes even CapHeight has very high value, but Ascent and Descent are ok
|
||||
float ascent = fontDescriptor.getAscent();
|
||||
float descent = fontDescriptor.getDescent();
|
||||
if (capHeight > ascent && ascent > 0 && descent < 0 && ((ascent - descent) / 2 < glyphHeight || Float.compare(glyphHeight, 0) == 0)) {
|
||||
glyphHeight = (ascent - descent) / 2;
|
||||
}
|
||||
}
|
||||
|
||||
// transformPoint from glyph space -> text space
|
||||
float height;
|
||||
if (font instanceof PDType3Font) {
|
||||
height = font.getFontMatrix().transformPoint(0, glyphHeight).y;
|
||||
} else {
|
||||
height = glyphHeight / 1000;
|
||||
}
|
||||
|
||||
return height;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A method provided as an event interface to allow a subclass to perform some specific
|
||||
* functionality when text needs to be processed.
|
||||
*
|
||||
* @param text The text to be processed.
|
||||
*/
|
||||
protected void processTextPosition(TextPosition text) {
|
||||
// subclasses can override to provide specific functionality
|
||||
}
|
||||
|
||||
}
|
||||
-82
@@ -1,82 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import org.apache.pdfbox.text.PDFTextStripperByArea;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class PDFAreaTextStripper extends PDFTextStripperByArea {
|
||||
|
||||
@Getter
|
||||
private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
|
||||
public PDFAreaTextStripper() throws IOException {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
|
||||
public void clearPositions() {
|
||||
|
||||
textPositionSequences = new ArrayList<>();
|
||||
}
|
||||
|
||||
}
|
||||
+65
-119
@@ -1,6 +1,38 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.contentstream.operator.OperatorName;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
|
||||
import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
|
||||
@@ -8,51 +40,30 @@ import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.apache.pdfbox.contentstream.operator.Operator;
|
||||
import org.apache.pdfbox.contentstream.operator.OperatorName;
|
||||
import org.apache.pdfbox.contentstream.operator.color.*;
|
||||
import org.apache.pdfbox.contentstream.operator.state.*;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.cos.COSBase;
|
||||
import org.apache.pdfbox.cos.COSNumber;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
@Getter
|
||||
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
@Getter
|
||||
private final List<Ruling> rulings = new ArrayList<>();
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
@Setter
|
||||
protected PDPage pdpage;
|
||||
@Getter
|
||||
private int minCharWidth;
|
||||
@Getter
|
||||
private int maxCharWidth;
|
||||
@Getter
|
||||
private int minCharHeight;
|
||||
private int maxCharWidths;
|
||||
|
||||
@Getter
|
||||
private int maxCharHeight;
|
||||
|
||||
@Getter
|
||||
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
|
||||
|
||||
@Getter
|
||||
private final List<Ruling> rulings = new ArrayList<>();
|
||||
|
||||
private final List<Ruling> graphicsPath = new ArrayList<>();
|
||||
|
||||
private float path_x;
|
||||
private float path_y;
|
||||
|
||||
@Setter
|
||||
private int pageNumber;
|
||||
|
||||
|
||||
public PDFLinesTextStripper() throws IOException {
|
||||
|
||||
super();
|
||||
this.addOperator(new SetStrokingColorSpace());
|
||||
this.addOperator(new SetNonStrokingColorSpace());
|
||||
@@ -76,9 +87,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
this.addOperator(new SetLineWidth());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments) throws IOException {
|
||||
protected void processOperator(Operator operator, List<COSBase> arguments)
|
||||
throws IOException {
|
||||
|
||||
String operation = operator.getName();
|
||||
|
||||
@@ -157,15 +168,12 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
case OperatorName.ENDPATH:
|
||||
graphicsPath.clear();
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
super.processOperator(operator, arguments);
|
||||
}
|
||||
|
||||
|
||||
private float floatValue(COSBase value) {
|
||||
|
||||
if (value instanceof COSNumber) {
|
||||
return ((COSNumber) value).floatValue();
|
||||
} else {
|
||||
@@ -173,138 +181,76 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Point2D.Float transformPosition(float x, float y) {
|
||||
|
||||
return super.transformedPoint(x, y);
|
||||
}
|
||||
|
||||
|
||||
private void addVisibleRulings(List<Ruling> path, boolean stroke) throws IOException {
|
||||
|
||||
try {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor()
|
||||
.toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
if (stroke && !getGraphicsState().getStrokingColor().isPattern() && getGraphicsState().getStrokingColor().toRGB() == 0 || !stroke && !getGraphicsState().getNonStrokingColor().isPattern() && getGraphicsState().getNonStrokingColor().toRGB() == 0) {
|
||||
rulings.addAll(path);
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
log.debug("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor()
|
||||
.getColorSpace()
|
||||
.getName() + " does not support toRGB");
|
||||
log.error("UnsupportedOperationException: " + getGraphicsState().getStrokingColor().getColorSpace().getName() + " or " + getGraphicsState().getNonStrokingColor().getColorSpace().getName() + " does not support toRGB");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
|
||||
|
||||
int startIndex = 0;
|
||||
RedTextPosition previous = null;
|
||||
|
||||
textPositions.sort(Comparator.comparing(TextPosition::getXDirAdj));
|
||||
|
||||
for (int i = 0; i <= textPositions.size() - 1; i++) {
|
||||
|
||||
if (!textPositionSequences.isEmpty()) {
|
||||
previous = textPositionSequences.get(textPositionSequences.size() - 1)
|
||||
.getTextPositions()
|
||||
.get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
|
||||
}
|
||||
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if (charWidth < minCharWidth) {
|
||||
minCharWidth = charWidth;
|
||||
}
|
||||
if (charWidth > maxCharWidth) {
|
||||
maxCharWidth = charWidth;
|
||||
}
|
||||
|
||||
int charHeight = (int) textPositions.get(i).getHeightDir();
|
||||
if (charHeight < minCharHeight) {
|
||||
minCharHeight = charHeight;
|
||||
}
|
||||
if (charWidth > maxCharHeight) {
|
||||
if(charHeight > maxCharHeight){
|
||||
maxCharHeight = charHeight;
|
||||
}
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
|
||||
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
|
||||
if(charWidth > maxCharWidths){
|
||||
maxCharWidths = charWidth;
|
||||
}
|
||||
|
||||
if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
|
||||
if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
|
||||
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
|
||||
if (i > 0 && textPositions.get(i).getUnicode().equals(" ") && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
}
|
||||
|
||||
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\t")) && i <= textPositions.size() - 2) {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
|
||||
// Remove false sequence ends (whitespaces)
|
||||
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
|
||||
.getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
|
||||
for (TextPosition t : sublist) {
|
||||
textPositionSequences.get(textPositionSequences.size() - 1).add(t);
|
||||
}
|
||||
} else {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
}
|
||||
startIndex = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
|
||||
if (!sublist.isEmpty() && sublist.get(sublist.size() - 1).getUnicode().equals(" ")) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
|
||||
.getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
|
||||
for (TextPosition t : sublist) {
|
||||
textPositionSequences.get(textPositionSequences.size() - 1).add(t);
|
||||
}
|
||||
} else {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && sublist.get(0).getUnicode().equals(" "))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
super.writeString(text);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getText(PDDocument doc) throws IOException {
|
||||
|
||||
minCharWidth = Integer.MAX_VALUE;
|
||||
maxCharWidth = 0;
|
||||
minCharHeight = Integer.MAX_VALUE;
|
||||
maxCharHeight = 0;
|
||||
maxCharWidths = 0;
|
||||
maxCharWidths = 0;
|
||||
textPositionSequences.clear();
|
||||
rulings.clear();
|
||||
graphicsPath.clear();
|
||||
@@ -315,4 +261,4 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
-2067
File diff suppressed because it is too large
Load Diff
+22
@@ -0,0 +1,22 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public class ParsedElements {
|
||||
|
||||
private List<TextPositionSequence> sequences;
|
||||
private List<Ruling> rulings;
|
||||
|
||||
private boolean landscape;
|
||||
private boolean rotated;
|
||||
|
||||
private float maxCharWidth;
|
||||
private float maxCharHeight;
|
||||
}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@CompiledJson
|
||||
public class RedTextPosition {
|
||||
|
||||
private String textMatrix;
|
||||
private float[] position;
|
||||
|
||||
@JsonIgnore
|
||||
private int rotation;
|
||||
|
||||
@JsonIgnore
|
||||
private float pageHeight;
|
||||
|
||||
@JsonIgnore
|
||||
private float pageWidth;
|
||||
|
||||
private String unicode;
|
||||
|
||||
@JsonIgnore
|
||||
private float dir;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
private float widthOfSpace;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
private float fontSizeInPt;
|
||||
|
||||
// not used in reanalysis
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
private String fontName;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public static RedTextPosition fromTextPosition(TextPosition textPosition) {
|
||||
|
||||
var pos = new RedTextPosition();
|
||||
BeanUtils.copyProperties(textPosition, pos);
|
||||
pos.setFontName(textPosition.getFont().getName());
|
||||
|
||||
pos.setFontSizeInPt(textPosition.getFontSizeInPt());
|
||||
|
||||
pos.setTextMatrix(textPosition.getTextMatrix().toString());
|
||||
|
||||
var position = new float[4];
|
||||
|
||||
position[0] = textPosition.getXDirAdj();
|
||||
position[1] = textPosition.getYDirAdj();
|
||||
position[2] = textPosition.getWidthDirAdj();
|
||||
position[3] = textPosition.getHeightDir();
|
||||
|
||||
pos.setPosition(position);
|
||||
return pos;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getXDirAdj() {
|
||||
|
||||
return position[0];
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getYDirAdj() {
|
||||
|
||||
return position[1];
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getWidthDirAdj() {
|
||||
|
||||
return position[2];
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
public float getHeightDir() {
|
||||
|
||||
return position[3];
|
||||
}
|
||||
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public enum TextDirection {
|
||||
ZERO(0f),
|
||||
QUARTER_CIRCLE(90f),
|
||||
HALF_CIRCLE(180f),
|
||||
THREE_QUARTER_CIRCLE(270f);
|
||||
|
||||
public static final String VALUE_STRING_SUFFIX = "°";
|
||||
|
||||
@JsonValue
|
||||
private final float degrees;
|
||||
private final float radians;
|
||||
|
||||
|
||||
TextDirection(float degreeValue) {
|
||||
|
||||
degrees = degreeValue;
|
||||
radians = (float) Math.toRadians(degreeValue);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return degrees + VALUE_STRING_SUFFIX;
|
||||
}
|
||||
|
||||
|
||||
@com.dslplatform.json.JsonValue
|
||||
public float jsonValue() {
|
||||
|
||||
return getDegrees();
|
||||
}
|
||||
|
||||
|
||||
@JsonCreator(mode = JsonCreator.Mode.DELEGATING)
|
||||
public static TextDirection fromDegrees(float degrees) {
|
||||
|
||||
for (var dir : TextDirection.values()) {
|
||||
if (degrees == dir.degrees) {
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException(String.format("A value of %f is not supported by TextDirection", degrees));
|
||||
}
|
||||
|
||||
|
||||
public static TextDirection fromString(String degreesAsString) {
|
||||
|
||||
Objects.requireNonNull(degreesAsString, "Cannot construct a text direction from a null value");
|
||||
|
||||
String value = degreesAsString.strip();
|
||||
|
||||
if (degreesAsString.endsWith(VALUE_STRING_SUFFIX)) {
|
||||
value = degreesAsString.replace(VALUE_STRING_SUFFIX + "$", "");
|
||||
}
|
||||
|
||||
return fromDegrees(Float.parseFloat(value));
|
||||
}
|
||||
}
|
||||
+82
-181
@@ -1,60 +1,29 @@
|
||||
package com.iqser.red.service.redaction.v1.server.parsing.model;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import com.dslplatform.json.JsonAttribute;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Slf4j
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonIgnoreProperties({"empty"})
|
||||
@RequiredArgsConstructor
|
||||
public class TextPositionSequence implements CharSequence {
|
||||
|
||||
public static final int HEIGHT_PADDING = 2;
|
||||
private int page;
|
||||
private List<RedTextPosition> textPositions = new ArrayList<>();
|
||||
private List<TextPosition> textPositions = new ArrayList<>();
|
||||
|
||||
private TextDirection dir;
|
||||
private int rotation;
|
||||
private float pageHeight;
|
||||
private float pageWidth;
|
||||
|
||||
|
||||
public TextPositionSequence(int page) {
|
||||
|
||||
this.page = page;
|
||||
}
|
||||
private final int page;
|
||||
|
||||
|
||||
public TextPositionSequence(List<TextPosition> textPositions, int page) {
|
||||
|
||||
this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
|
||||
this.textPositions = textPositions;
|
||||
this.page = page;
|
||||
this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
|
||||
this.rotation = textPositions.get(0).getRotation();
|
||||
this.pageHeight = textPositions.get(0).getPageHeight();
|
||||
this.pageWidth = textPositions.get(0).getPageWidth();
|
||||
}
|
||||
|
||||
|
||||
@@ -68,7 +37,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@Override
|
||||
public char charAt(int index) {
|
||||
|
||||
RedTextPosition textPosition = textPositionAt(index);
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return text.charAt(0);
|
||||
}
|
||||
@@ -76,7 +45,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
public char charAt(int index, boolean caseInSensitive) {
|
||||
|
||||
RedTextPosition textPosition = textPositionAt(index);
|
||||
TextPosition textPosition = textPositionAt(index);
|
||||
String text = textPosition.getUnicode();
|
||||
return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
|
||||
}
|
||||
@@ -85,15 +54,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@Override
|
||||
public TextPositionSequence subSequence(int start, int end) {
|
||||
|
||||
var textPositionSequence = new TextPositionSequence();
|
||||
textPositionSequence.textPositions = textPositions.subList(start, end);
|
||||
textPositionSequence.page = page;
|
||||
textPositionSequence.dir = dir;
|
||||
textPositionSequence.rotation = rotation;
|
||||
textPositionSequence.pageHeight = pageHeight;
|
||||
textPositionSequence.pageWidth = pageWidth;
|
||||
|
||||
return textPositionSequence;
|
||||
return new TextPositionSequence(textPositions.subList(start, end), page);
|
||||
}
|
||||
|
||||
|
||||
@@ -108,132 +69,94 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
public RedTextPosition textPositionAt(int index) {
|
||||
public TextPosition textPositionAt(int index) {
|
||||
|
||||
return textPositions.get(index);
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPositionSequence textPositionSequence, RedTextPosition textPosition) {
|
||||
|
||||
this.textPositions.add(textPosition);
|
||||
this.page = textPositionSequence.getPage();
|
||||
this.dir = textPositionSequence.getDir();
|
||||
this.rotation = textPositionSequence.getRotation();
|
||||
this.pageHeight = textPositionSequence.getPageHeight();
|
||||
this.pageWidth = textPositionSequence.getPageWidth();
|
||||
}
|
||||
|
||||
|
||||
public void add(TextPosition textPosition) {
|
||||
|
||||
this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
|
||||
|
||||
this.dir = TextDirection.fromDegrees(textPositions.get(0).getDir());
|
||||
this.rotation = textPositions.get(0).getRotation();
|
||||
this.pageHeight = textPositions.get(0).getPageHeight();
|
||||
this.pageWidth = textPositions.get(0).getPageWidth();
|
||||
|
||||
this.textPositions.add(textPosition);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
* This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
|
||||
*
|
||||
* @return the text direction adjusted minX value
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getMinXDirAdj() {
|
||||
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
public float getX1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
} else {
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
* This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
|
||||
*
|
||||
* @return the text direction adjusted maxX value
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getMaxXDirAdj() {
|
||||
public float getX2() {
|
||||
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
} else {
|
||||
return textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
}
|
||||
}
|
||||
|
||||
public float getRotationAdjustedY() {
|
||||
return textPositions.get(0).getY();
|
||||
}
|
||||
|
||||
public float getY1() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getXDirAdj();
|
||||
} else {
|
||||
return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
* This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
|
||||
*
|
||||
* @return the text direction adjusted minY value. The upper border of the bounding box of the word.
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getMinYDirAdj() {
|
||||
|
||||
return textPositions.get(0).getYDirAdj() - getTextHeight();
|
||||
public float getY2() {
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - 2;
|
||||
} else {
|
||||
return textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() + getTextHeight();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This value is adjusted so that 0,0 is upper left and it is adjusted based on the text direction.
|
||||
* This method ignores the page rotation but takes the text rotation and adjusts the coordinates to awt.
|
||||
*
|
||||
* @return the text direction adjusted maxY value. The lower border of the bounding box of the word.
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getMaxYDirAdj() {
|
||||
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getTextHeight() {
|
||||
|
||||
return textPositions.get(0).getHeightDir() + HEIGHT_PADDING;
|
||||
return textPositions.get(0).getHeightDir() + 2;
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getHeight() {
|
||||
|
||||
return getMaxYDirAdj() - getMinYDirAdj();
|
||||
return getY2() - getY1();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getWidth() {
|
||||
|
||||
return getMaxXDirAdj() - getMinXDirAdj();
|
||||
return getX2() - getX1();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public String getFont() {
|
||||
|
||||
return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", "");
|
||||
return textPositions.get(0)
|
||||
.getFont()
|
||||
.toString()
|
||||
.toLowerCase()
|
||||
.replaceAll(",bold", "")
|
||||
.replaceAll(",italic", "");
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public String getFontStyle() {
|
||||
|
||||
String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase();
|
||||
String lowercaseFontName = textPositions.get(0).getFont().toString().toLowerCase();
|
||||
|
||||
if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
|
||||
return "bold, italic";
|
||||
@@ -248,69 +171,47 @@ public class TextPositionSequence implements CharSequence {
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getFontSize() {
|
||||
|
||||
return textPositions.get(0).getFontSizeInPt();
|
||||
}
|
||||
|
||||
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
public float getSpaceWidth() {
|
||||
|
||||
return textPositions.get(0).getWidthOfSpace();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This returns the bounding box of the word in Pdf Coordinate System where {0,0} rotated with the page rotation.
|
||||
* 0 -> LowerLeft
|
||||
* 90 -> UpperLeft
|
||||
* 180 -> UpperRight
|
||||
* 270 -> LowerRight
|
||||
*
|
||||
* @return bounding box of the word in Pdf Coordinate System
|
||||
*/
|
||||
@JsonIgnore
|
||||
@JsonAttribute(ignore = true)
|
||||
@SneakyThrows
|
||||
public Rectangle getRectangle() {
|
||||
public int getRotation() {
|
||||
|
||||
log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, rotation, dir);
|
||||
|
||||
float textHeight = getTextHeight();
|
||||
|
||||
RedTextPosition firstTextPos = textPositions.get(0);
|
||||
RedTextPosition lastTextPos = textPositions.get(textPositions.size() - 1);
|
||||
|
||||
Point2D bottomLeft = new Point2D.Double(firstTextPos.getXDirAdj(), firstTextPos.getYDirAdj() - HEIGHT_PADDING);
|
||||
Point2D topRight = new Point2D.Double(lastTextPos.getXDirAdj() + lastTextPos.getWidthDirAdj(), lastTextPos.getYDirAdj() + textHeight + HEIGHT_PADDING);
|
||||
|
||||
AffineTransform transform = new AffineTransform();
|
||||
if (dir == TextDirection.ZERO || dir == TextDirection.HALF_CIRCLE) {
|
||||
transform.rotate(dir.getRadians(), pageWidth / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageHeight + textHeight);
|
||||
transform.scale(1., -1.);
|
||||
} else if (dir == TextDirection.QUARTER_CIRCLE) {
|
||||
transform.rotate(dir.getRadians(), pageWidth / 2f, pageWidth / 2f);
|
||||
transform.translate(0f, pageWidth + textHeight);
|
||||
transform.scale(1., -1.);
|
||||
} else {
|
||||
transform.rotate(dir.getRadians(), pageHeight / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageWidth + textHeight);
|
||||
transform.scale(1., -1.);
|
||||
}
|
||||
|
||||
bottomLeft = transform.transform(bottomLeft, null);
|
||||
topRight = transform.transform(topRight, null);
|
||||
|
||||
return new Rectangle( //
|
||||
new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()),
|
||||
(float) (topRight.getX() - bottomLeft.getX()),
|
||||
(float) (topRight.getY() - bottomLeft.getY()),
|
||||
page);
|
||||
return textPositions.get(0).getRotation();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Rectangle getRectangle() {
|
||||
|
||||
float height = getTextHeight();
|
||||
|
||||
float posXInit = getX1();
|
||||
float posXEnd;
|
||||
float posYInit;
|
||||
float posYEnd;
|
||||
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
posXEnd = textPositions.get(0).getYDirAdj() + 2;
|
||||
posYInit = getY1();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
|
||||
} else {
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + 2;
|
||||
}
|
||||
|
||||
return new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class MessageReceiver {
|
||||
|
||||
private final RedactionMessageReceiver redactionMessageReceiver;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_QUEUE)
|
||||
public void receiveAnalyzeRequest(Message message) {
|
||||
|
||||
redactionMessageReceiver.receiveAnalyzeRequest(message, false);
|
||||
}
|
||||
|
||||
}
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class MessagingConfiguration {
|
||||
|
||||
public static final String REDACTION_QUEUE = "redactionQueue";
|
||||
public static final String REDACTION_DQL = "redactionDQL";
|
||||
|
||||
public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
|
||||
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
|
||||
public MessageReceiver messageReceiver(RedactionMessageReceiver redactionMessageReceiver) {
|
||||
|
||||
return new MessageReceiver(redactionMessageReceiver);
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
|
||||
public PriorityMessageReceiver priorityMessageReceiver(RedactionMessageReceiver redactionMessageReceiver) {
|
||||
|
||||
return new PriorityMessageReceiver(redactionMessageReceiver);
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionPriorityQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionDeadLetterQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_DQL).build();
|
||||
}
|
||||
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class PriorityMessageReceiver {
|
||||
|
||||
private final RedactionMessageReceiver redactionMessageReceiver;
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
|
||||
public void receiveAnalyzeRequest(Message message) {
|
||||
|
||||
redactionMessageReceiver.receiveAnalyzeRequest(message, true);
|
||||
}
|
||||
|
||||
}
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
|
||||
import org.springframework.amqp.AmqpRejectAndDontRequeueException;
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.analyze.AnalyzeService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionMessageReceiver {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AnalyzeService analyzeService;
|
||||
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
|
||||
private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void receiveAnalyzeRequest(Message message, boolean priority) {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(message.getBody(), AnalyzeRequest.class);
|
||||
|
||||
// This prevents from endless retries oom errors.
|
||||
if (message.getMessageProperties().isRedelivered()) {
|
||||
throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
|
||||
analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId()));
|
||||
}
|
||||
|
||||
receiveAnalyzeRequest(analyzeRequest, priority);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void receiveAnalyzeRequest(AnalyzeRequest analyzeRequest, boolean priority) {
|
||||
|
||||
log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
|
||||
AnalyzeResult result;
|
||||
|
||||
try {
|
||||
switch (analyzeRequest.getMessageType()) {
|
||||
|
||||
case REANALYSE:
|
||||
result = analyzeService.reanalyze(analyzeRequest);
|
||||
log.info("Successfully reanalyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
case STRUCTURE_ANALYSE:
|
||||
result = analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||
log.info("Successfully analyzed structure dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
case ANALYSE:
|
||||
result = analyzeService.analyze(analyzeRequest);
|
||||
log.info("Successfully analyzed dossier {} file {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
analyzeRequest.getManualRedactions());
|
||||
log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}",
|
||||
analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
result.getDuration());
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
|
||||
}
|
||||
|
||||
result.setMessageType(analyzeRequest.getMessageType());
|
||||
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_DQL)
|
||||
public void receiveAnalyzeRequestDQL(String in) throws JsonProcessingException {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
|
||||
log.info("Failed to process analyze request: {}", analyzeRequest);
|
||||
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
|
||||
}
|
||||
+17
-30
@@ -4,22 +4,14 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.Value;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@Value
|
||||
public class CellValue {
|
||||
|
||||
private List<TextBlock> textBlocks = new ArrayList<>();
|
||||
TextBlock textBlock;
|
||||
|
||||
private int rowSpanStart;
|
||||
int rowSpanStart;
|
||||
|
||||
|
||||
@Override
|
||||
@@ -27,27 +19,22 @@ public class CellValue {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
Iterator<TextBlock> itty = textBlocks.iterator();
|
||||
while (itty.hasNext()) {
|
||||
TextBlock textBlock = itty.next();
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
TextPositionSequence previous = null;
|
||||
for (TextPositionSequence word : textBlock.getSequences()) {
|
||||
if (previous != null) {
|
||||
if (Math.abs(previous.getRotationAdjustedY() - word.getRotationAdjustedY()) > word.getTextHeight()) {
|
||||
sb.append('\n');
|
||||
} else {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
if (itty.hasNext()) {
|
||||
sb.append(' ');
|
||||
}
|
||||
sb.append(word.toString());
|
||||
previous = word;
|
||||
}
|
||||
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
|
||||
return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll(" {2}", " ");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+25
-21
@@ -1,39 +1,34 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class Dictionary {
|
||||
|
||||
public static final String RECOMMENDATION_PREFIX = "recommendation_";
|
||||
|
||||
@Getter
|
||||
private List<DictionaryModel> dictionaryModels;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
private Map<String, DictionaryModel> localAccessMap;
|
||||
|
||||
@Getter
|
||||
private DictionaryVersion version;
|
||||
private long version;
|
||||
|
||||
|
||||
public Dictionary(List<DictionaryModel> dictionaryModels, DictionaryVersion version) {
|
||||
public boolean isRecommendation(String type) {
|
||||
|
||||
this.dictionaryModels = dictionaryModels;
|
||||
this.dictionaryModels.forEach(dm -> localAccessMap.put(dm.getType(), dm));
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
|
||||
public int getDictionaryRank(String type) {
|
||||
|
||||
if (!localAccessMap.containsKey(type)) {
|
||||
return 0;
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model != null) {
|
||||
return model.isRecommendation();
|
||||
}
|
||||
return localAccessMap.get(type).getRank();
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -49,10 +44,20 @@ public class Dictionary {
|
||||
}
|
||||
|
||||
|
||||
public DictionaryModel getType(String type) {
|
||||
DictionaryModel model = localAccessMap.get(type);
|
||||
if (model == null) throw new NotFoundException("Type: " + type + " is not found");
|
||||
return model;
|
||||
public boolean containsValue(String type, String value) {
|
||||
|
||||
if (localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(type) && localAccessMap.get(type)
|
||||
.getLocalEntries()
|
||||
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
|
||||
.getEntries()
|
||||
.contains(value) || localAccessMap.containsKey(RECOMMENDATION_PREFIX + type) && localAccessMap.get(RECOMMENDATION_PREFIX + type)
|
||||
.getLocalEntries()
|
||||
.contains(value)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -65,7 +70,6 @@ public class Dictionary {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public boolean isCaseInsensitiveDictionary(String type) {
|
||||
|
||||
DictionaryModel dictionaryModel = localAccessMap.get(type);
|
||||
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DictionaryEntries {
|
||||
|
||||
@Builder.Default
|
||||
Set<DictionaryEntry> entries = new HashSet<>();
|
||||
@Builder.Default
|
||||
Set<DictionaryEntry> falsePositives = new HashSet<>();
|
||||
@Builder.Default
|
||||
Set<DictionaryEntry> falseRecommendations = new HashSet<>();
|
||||
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryIncrement {
|
||||
|
||||
private Set<DictionaryIncrementValue> values;
|
||||
private DictionaryVersion dictionaryVersion;
|
||||
|
||||
}
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryIncrementValue {
|
||||
|
||||
private String value;
|
||||
private boolean caseinsensitive;
|
||||
|
||||
}
|
||||
+13
-98
@@ -1,111 +1,26 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public class DictionaryModel implements Serializable {
|
||||
public class DictionaryModel {
|
||||
|
||||
private final String type;
|
||||
private final int rank;
|
||||
private final float[] color;
|
||||
private final boolean caseInsensitive;
|
||||
private final boolean hint;
|
||||
private final boolean isDossierDictionary;
|
||||
private final Set<DictionaryEntry> entries;
|
||||
private final Set<DictionaryEntry> falsePositives;
|
||||
private final Set<DictionaryEntry> falseRecommendations;
|
||||
private String type;
|
||||
private int rank;
|
||||
private float[] color;
|
||||
private boolean caseInsensitive;
|
||||
private boolean hint;
|
||||
private boolean recommendation;
|
||||
private Set<String> entries;
|
||||
private Set<String> localEntries;
|
||||
|
||||
private transient SearchImplementation entriesSearch;
|
||||
private transient SearchImplementation falsePositiveSearch;
|
||||
private transient SearchImplementation falseRecommendationsSearch;
|
||||
private transient SearchImplementation localSearch;
|
||||
|
||||
private final Set<String> localEntries = new HashSet<>();
|
||||
|
||||
|
||||
public DictionaryModel(String type,
|
||||
int rank,
|
||||
float[] color,
|
||||
boolean caseInsensitive,
|
||||
boolean hint,
|
||||
Set<DictionaryEntry> entries,
|
||||
Set<DictionaryEntry> falsePositives,
|
||||
Set<DictionaryEntry> falseRecommendations,
|
||||
boolean isDossierDictionary) {
|
||||
|
||||
this.type = type;
|
||||
this.rank = rank;
|
||||
this.color = color;
|
||||
this.caseInsensitive = caseInsensitive;
|
||||
this.hint = hint;
|
||||
this.isDossierDictionary = isDossierDictionary;
|
||||
|
||||
this.entries = entries;
|
||||
this.falsePositives = falsePositives;
|
||||
this.falseRecommendations = falseRecommendations;
|
||||
|
||||
this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()),
|
||||
caseInsensitive);
|
||||
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()),
|
||||
caseInsensitive);
|
||||
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.collect(Collectors.toList()), caseInsensitive);
|
||||
|
||||
}
|
||||
|
||||
|
||||
public SearchImplementation getLocalSearch() {
|
||||
|
||||
if (this.localSearch == null) {
|
||||
this.localSearch = new SearchImplementation(this.localEntries, caseInsensitive);
|
||||
}
|
||||
return this.localSearch;
|
||||
}
|
||||
|
||||
|
||||
public SearchImplementation getEntriesSearch() {
|
||||
|
||||
if (entriesSearch == null) {
|
||||
this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()),
|
||||
caseInsensitive);
|
||||
}
|
||||
return entriesSearch;
|
||||
}
|
||||
|
||||
|
||||
public SearchImplementation getFalsePositiveSearch() {
|
||||
|
||||
if (falsePositiveSearch == null) {
|
||||
this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.collect(Collectors.toList()), caseInsensitive);
|
||||
}
|
||||
return falsePositiveSearch;
|
||||
}
|
||||
|
||||
|
||||
public SearchImplementation getFalseRecommendationsSearch() {
|
||||
|
||||
if (falseRecommendationsSearch == null) {
|
||||
this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
|
||||
.filter(e -> !e.isDeleted())
|
||||
.map(DictionaryEntry::getValue)
|
||||
.collect(Collectors.toList()), caseInsensitive);
|
||||
}
|
||||
return falseRecommendationsSearch;
|
||||
public Set<String> getValues(boolean local){
|
||||
return local ? localEntries : entries;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
public class DictionaryRepresentation {
|
||||
|
||||
private String dossierTemplateId;
|
||||
private long dictionaryVersion = -1;
|
||||
private List<DictionaryModel> dictionary = new ArrayList<>();
|
||||
private float[] defaultColor;
|
||||
private float[] requestAddColor;
|
||||
private float[] requestRemoveColor;
|
||||
private float[] notRedactedColor;
|
||||
private Map<String, DictionaryModel> localAccessMap = new HashMap<>();
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DictionaryVersion {
|
||||
|
||||
long dossierTemplateVersion;
|
||||
long dossierVersion;
|
||||
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Entities {
|
||||
|
||||
@Builder.Default
|
||||
private Set<Entity> entities = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<Entity> nerEntities = new HashSet<>();
|
||||
|
||||
}
|
||||
+11
-77
@@ -1,34 +1,26 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity implements ReasonHolder {
|
||||
public class Entity {
|
||||
|
||||
private String word;
|
||||
private String type;
|
||||
@EqualsAndHashCode.Include
|
||||
private final String word;
|
||||
private final String type;
|
||||
private boolean redaction;
|
||||
private boolean falsePositive;
|
||||
private String redactionReason;
|
||||
private String legalBasis;
|
||||
private List<EntityPositionSequence> positionSequences = new ArrayList<>();
|
||||
private List<TextPositionSequence> targetSequences;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer start;
|
||||
@EqualsAndHashCode.Include
|
||||
private Integer end;
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
@@ -38,44 +30,8 @@ public class Entity implements ReasonHolder {
|
||||
@EqualsAndHashCode.Include
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
private boolean ignored;
|
||||
|
||||
private Set<Engine> engines = new HashSet<>();
|
||||
|
||||
private Set<Entity> references = new HashSet<>();
|
||||
|
||||
private EntityType entityType;
|
||||
|
||||
private boolean resized;
|
||||
|
||||
private boolean skipRemoveEntitiesContainedInLarger;
|
||||
|
||||
|
||||
public Entity(String word,
|
||||
String type,
|
||||
boolean redaction,
|
||||
String redactionReason,
|
||||
List<EntityPositionSequence> positionSequences,
|
||||
String headline,
|
||||
int matchedRule,
|
||||
int sectionNumber,
|
||||
String legalBasis,
|
||||
boolean isDictionaryEntry,
|
||||
String textBefore,
|
||||
String textAfter,
|
||||
Integer start,
|
||||
Integer end,
|
||||
boolean isDossierDictionaryEntry,
|
||||
Set<Engine> engines,
|
||||
Set<Entity> references,
|
||||
EntityType entityType) {
|
||||
public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -86,28 +42,10 @@ public class Entity implements ReasonHolder {
|
||||
this.matchedRule = matchedRule;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.legalBasis = legalBasis;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.textBefore = textBefore;
|
||||
this.textAfter = textAfter;
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
this.engines = engines;
|
||||
this.references = references;
|
||||
this.entityType = entityType;
|
||||
}
|
||||
|
||||
|
||||
public Entity(String word,
|
||||
String type,
|
||||
Integer start,
|
||||
Integer end,
|
||||
String headline,
|
||||
int sectionNumber,
|
||||
boolean isDictionaryEntry,
|
||||
boolean isDossierDictionaryEntry,
|
||||
Engine engine,
|
||||
EntityType entityType) {
|
||||
public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
|
||||
|
||||
this.word = word;
|
||||
this.type = type;
|
||||
@@ -115,10 +53,6 @@ public class Entity implements ReasonHolder {
|
||||
this.end = end;
|
||||
this.headline = headline;
|
||||
this.sectionNumber = sectionNumber;
|
||||
this.isDictionaryEntry = isDictionaryEntry;
|
||||
this.isDossierDictionaryEntry = isDossierDictionaryEntry;
|
||||
this.engines.add(engine);
|
||||
this.entityType = entityType;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-3
@@ -6,21 +6,19 @@ import java.util.List;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@RequiredArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class EntityPositionSequence {
|
||||
|
||||
private final String id;
|
||||
@EqualsAndHashCode.Exclude
|
||||
private List<TextPositionSequence> sequences = new ArrayList<>();
|
||||
private int pageNumber;
|
||||
private final String id;
|
||||
|
||||
}
|
||||
|
||||
-8
@@ -1,8 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
public enum EntityType {
|
||||
ENTITY,
|
||||
RECOMMENDATION,
|
||||
FALSE_POSITIVE,
|
||||
FALSE_RECOMMENDATION
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class FindEntitiesResult {
|
||||
|
||||
private Set<Entity> entities;
|
||||
private Set<FileAttribute> addedFileAttributes;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user