Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
712b86a19a | ||
|
|
c54ee6e7bf | ||
|
|
a4d41fc164 | ||
|
|
e1b8f5b89a | ||
|
|
5834e8891f | ||
|
|
9c8e2f047e | ||
|
|
24612e3471 | ||
|
|
9be8d46f4e | ||
|
|
3e32540616 | ||
|
|
5403829fd8 | ||
|
|
3e53e156a5 | ||
|
|
4c73d2841b | ||
|
|
db8eedc9a3 |
@@ -26,4 +26,3 @@
|
||||
**/.DS_Store
|
||||
**/classpath-data.json
|
||||
**/dependencies-and-licenses-overview.txt
|
||||
/redaction-service-v1/redaction-service-server-v1/src/test/resources/RedactionLog/
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package buildjob;
|
||||
|
||||
import java.time.DayOfWeek;
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
|
||||
import java.time.LocalTime;
|
||||
|
||||
import com.atlassian.bamboo.specs.api.BambooSpec;
|
||||
import com.atlassian.bamboo.specs.api.builders.BambooKey;
|
||||
import com.atlassian.bamboo.specs.api.builders.Variable;
|
||||
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
|
||||
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
|
||||
@@ -17,20 +19,16 @@ import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
|
||||
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
|
||||
import com.atlassian.bamboo.specs.api.builders.project.Project;
|
||||
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
|
||||
import com.atlassian.bamboo.specs.api.builders.Variable;
|
||||
import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
|
||||
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
|
||||
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
|
||||
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
|
||||
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
|
||||
|
||||
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
|
||||
import com.atlassian.bamboo.specs.util.BambooServer;
|
||||
|
||||
/**
|
||||
* Plan configuration for Bamboo.
|
||||
@@ -41,14 +39,15 @@ public class PlanSpec {
|
||||
|
||||
private static final String SERVICE_NAME = "redaction-service";
|
||||
|
||||
private static final String JVM_ARGS =" -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
|
||||
private static final String JVM_ARGS = " -Xmx4g -XX:+ExitOnOutOfMemoryError -XX:SurvivorRatio=2 -XX:NewRatio=1 -XX:InitialTenuringThreshold=16 -XX:MaxTenuringThreshold=16 -XX:InitiatingHeapOccupancyPercent=35 ";
|
||||
|
||||
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "");
|
||||
|
||||
|
||||
/**
|
||||
* Run main to publish plan on Bamboo
|
||||
*/
|
||||
public static void main(final String[] args) throws Exception {
|
||||
public static void main(final String[] args) {
|
||||
//By default credentials are read from the '.credentials' file.
|
||||
BambooServer bambooServer = new BambooServer("http://localhost:8085");
|
||||
|
||||
@@ -57,15 +56,21 @@ public class PlanSpec {
|
||||
PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
|
||||
bambooServer.publish(planPermission);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
bambooServer.publish(nightPlan);
|
||||
PlanPermissions nightPlanPermission = new PlanSpec().createPlanPermission(nightPlan.getIdentifier());
|
||||
bambooServer.publish(nightPlanPermission);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
bambooServer.publish(secPlan);
|
||||
PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
|
||||
bambooServer.publish(secPlanPermission);
|
||||
}
|
||||
|
||||
|
||||
private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
|
||||
Permissions permission = new Permissions()
|
||||
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
|
||||
|
||||
Permissions permission = new Permissions().userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("devplant", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.loggedInUserPermissions(PermissionType.VIEW)
|
||||
@@ -73,105 +78,80 @@ public class PlanSpec {
|
||||
return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
|
||||
}
|
||||
|
||||
|
||||
private Project project() {
|
||||
return new Project()
|
||||
.name("RED")
|
||||
.key(new BambooKey("RED"));
|
||||
|
||||
return new Project().name("RED").key(new BambooKey("RED"));
|
||||
}
|
||||
|
||||
|
||||
public Plan createPlan() {
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME, new BambooKey(SERVICE_KEY))
|
||||
.description("Plan created from (enter repository url of your plan)")
|
||||
|
||||
return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Plan created from (enter repository url of your plan)")
|
||||
.variables(new Variable("maven_add_param", ""))
|
||||
.stages(new Stage("Default Stage")
|
||||
.jobs(new Job("Default Job",
|
||||
new BambooKey("JOB1"))
|
||||
.tasks(
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
"rm -rf ./*"),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME),
|
||||
createJUnitParserTask()
|
||||
.description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true),
|
||||
new InjectVariablesTask()
|
||||
.description("Inject git Tag")
|
||||
.path("git.tag")
|
||||
.namespace("g")
|
||||
.scope(InjectVariablesScope.LOCAL),
|
||||
new VcsTagTask()
|
||||
.description("${bamboo.g.gitTag}")
|
||||
.tagName("${bamboo.g.gitTag}")
|
||||
.defaultRepository())
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock")
|
||||
)
|
||||
)
|
||||
)
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true), new InjectVariablesTask().description("Inject git Tag")
|
||||
.path("git.tag")
|
||||
.namespace("g")
|
||||
.scope(InjectVariablesScope.LOCAL), new VcsTagTask().description("${bamboo.g.gitTag}").tagName("${bamboo.g.gitTag}").defaultRepository())
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
|
||||
.triggers(new BitbucketServerTrigger())
|
||||
.planBranchManagement(new PlanBranchManagement()
|
||||
.createForVcsBranch()
|
||||
.delete(new BranchCleanup()
|
||||
.whenInactiveInRepositoryAfterDays(14))
|
||||
.planBranchManagement(new PlanBranchManagement().createForVcsBranch()
|
||||
.delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14))
|
||||
.notificationForCommitters());
|
||||
}
|
||||
|
||||
public Plan createSecBuild() {
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
|
||||
.description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage")
|
||||
.jobs(new Job("Default Job",
|
||||
new BambooKey("JOB1"))
|
||||
.tasks(
|
||||
new ScriptTask()
|
||||
.description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" +
|
||||
"set -e\n" +
|
||||
"rm -rf ./*"),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Sonar")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
|
||||
.argument(SERVICE_NAME))
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.dockerRunArguments("--net=host")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock")
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
public Plan createNightPlan() {
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Night", new BambooKey(SERVICE_KEY + "NIGHT")).description("Long running nightly Plan for tests")
|
||||
.variables(new Variable("maven_add_param", "-Dtest-groups=rules-test"))
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new CleanWorkingDirectoryTask().description("Clean working directory.")
|
||||
.enabled(true), new VcsCheckoutTask().description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Build")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/build-java.sh")
|
||||
.argument(SERVICE_NAME), createJUnitParserTask().description("Resultparser")
|
||||
.resultDirectories("**/test-reports/*.xml, **/target/surefire-reports/*.xml, **/target/failsafe-reports/*.xml")
|
||||
.enabled(true))
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
.triggers(
|
||||
new ScheduledTrigger()
|
||||
.scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement()
|
||||
.createForVcsBranchMatching("release.*")
|
||||
.notificationForCommitters());
|
||||
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement().delete(new BranchCleanup().whenInactiveInRepositoryAfterDays(14)).notificationForCommitters());
|
||||
}
|
||||
|
||||
|
||||
public Plan createSecBuild() {
|
||||
|
||||
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage").jobs(new Job("Default Job", new BambooKey("JOB1")).tasks(new ScriptTask().description("Clean")
|
||||
.inlineBody("#!/bin/bash\n" + "set -e\n" + "rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository")
|
||||
.cleanCheckout(true)
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()), new ScriptTask().description("Sonar")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
|
||||
.argument(SERVICE_NAME))
|
||||
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
|
||||
.dockerRunArguments("--net=host")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RED / " + SERVICE_NAME)
|
||||
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(new PlanBranchManagement().createForVcsBranchMatching("release.*").notificationForCommitters());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -10,9 +10,13 @@ import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
|
||||
public class PlanSpecTest {
|
||||
@Test
|
||||
public void checkYourPlanOffline() throws PropertiesValidationException {
|
||||
|
||||
Plan plan = new PlanSpec().createPlan();
|
||||
EntityPropertiesBuilders.build(plan);
|
||||
|
||||
Plan nightPlan = new PlanSpec().createNightPlan();
|
||||
EntityPropertiesBuilders.build(nightPlan);
|
||||
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
EntityPropertiesBuilders.build(secPlan);
|
||||
}
|
||||
|
||||
@@ -29,15 +29,10 @@
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.dslplatform</groupId>
|
||||
<artifactId>dsl-json-java8</artifactId>
|
||||
<version>${dsljson.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-commons-dependency</artifactId>
|
||||
<version>1.15.0</version>
|
||||
<version>1.17.0</version>
|
||||
<scope>import</scope>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
|
||||
+2
@@ -1,5 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
@@ -11,6 +12,7 @@ import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@CompiledJson
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImportedRedactions {
|
||||
|
||||
+7
-7
@@ -1,16 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
public class MockMultipartFile implements MultipartFile {
|
||||
|
||||
private final String name;
|
||||
@@ -82,13 +82,13 @@ public class MockMultipartFile implements MultipartFile {
|
||||
}
|
||||
|
||||
|
||||
public byte[] getBytes() throws IOException {
|
||||
public byte[] getBytes() {
|
||||
|
||||
return this.content;
|
||||
}
|
||||
|
||||
|
||||
public InputStream getInputStream() throws IOException {
|
||||
public InputStream getInputStream() {
|
||||
|
||||
return new ByteArrayInputStream(this.content);
|
||||
}
|
||||
|
||||
+2
-2
@@ -5,8 +5,8 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.dslplatform.json.CompiledJson;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@@ -16,6 +16,6 @@ import lombok.NoArgsConstructor;
|
||||
@AllArgsConstructor
|
||||
public class NerEntities {
|
||||
|
||||
private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
|
||||
private Map<Integer, List<EntityRecogintionEntity>> data = new HashMap<>();
|
||||
|
||||
}
|
||||
|
||||
+567
-683
File diff suppressed because it is too large
Load Diff
+21
-42
@@ -137,8 +137,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
if (textPositions.get(0).getRotation() == 90) {
|
||||
return textPositions.get(0).getYDirAdj();
|
||||
} else {
|
||||
return textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidth() + 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,8 +261,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
@JsonAttribute(ignore = true)
|
||||
public Rectangle getRectangle() {
|
||||
|
||||
log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0)
|
||||
.getRotation(), textPositions.get(0).getDir());
|
||||
log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, this, textPositions.get(0).getRotation(), textPositions.get(0).getDir());
|
||||
|
||||
float height = getTextHeight();
|
||||
|
||||
@@ -275,36 +273,30 @@ public class TextPositionSequence implements CharSequence {
|
||||
if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 90f) {
|
||||
|
||||
posYInit = getX1();
|
||||
posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1)
|
||||
.getWidthDirAdj() - 3;
|
||||
posYEnd = getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posXInit = textPositions.get(0).getYDirAdj() + 2;
|
||||
posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 180f) {
|
||||
|
||||
posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0)
|
||||
.getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posYInit = textPositions.get(0).getYDirAdj() - height + 2;
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
|
||||
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
|
||||
.getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + height;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 0.0f) {
|
||||
|
||||
posXInit = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
|
||||
posXInit = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getHeightDir();
|
||||
posXEnd = textPositions.get(0).getXDirAdj();
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 90.0f) {
|
||||
posXEnd = textPositions.get(0).getYDirAdj() + 2;
|
||||
@@ -313,29 +305,23 @@ public class TextPositionSequence implements CharSequence {
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 180.0f) {
|
||||
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() - 4;
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getXDirAdj() - 4;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getXDirAdj();
|
||||
posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1)
|
||||
.getHeightDir();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
|
||||
posYInit = textPositions.get(0).getYDirAdj() - 2 - textPositions.get(textPositions.size() - 1).getHeightDir();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - textPositions.get(textPositions.size() - 1).getHeightDir();
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 90 && textPositions.get(0).getDir() == 270.0f) {
|
||||
|
||||
posXInit = textPositions.get(0).getPageWidth() - getX1();
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYInit = textPositions.get(0).getPageHeight() - getY1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() - height - 4;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getXDirAdj() - height - 4;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 0f) {
|
||||
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
|
||||
posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 90f) {
|
||||
|
||||
@@ -347,27 +333,23 @@ public class TextPositionSequence implements CharSequence {
|
||||
} else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) {
|
||||
|
||||
posXInit = textPositions.get(0).getPageWidth() - getX1() + 1;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0)
|
||||
.getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj() - 3;
|
||||
posYInit = textPositions.get(0).getYDirAdj() - height + 2;
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height + 2;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 270.0f) {
|
||||
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
|
||||
.getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1).getWidthDirAdj();
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + height;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
|
||||
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = posYInit + 1;
|
||||
posXInit = textPositions.get(0).getXDirAdj();
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
|
||||
posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 90.0f) {
|
||||
|
||||
@@ -388,16 +370,13 @@ public class TextPositionSequence implements CharSequence {
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - height;
|
||||
posXInit = textPositions.get(0).getPageWidth() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + height;
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1).getYDirAdj() + height;
|
||||
|
||||
} else {
|
||||
// page rotation = 0 and text direction = 0
|
||||
posXEnd = textPositions.get(textPositions.size() - 1)
|
||||
.getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
|
||||
posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 1;
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + 2;
|
||||
posYEnd = textPositions.get(0).getPageHeight() - textPositions.get(textPositions.size() - 1).getYDirAdj() + 2;
|
||||
}
|
||||
|
||||
var rectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);
|
||||
|
||||
+10
-5
@@ -1,5 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
@@ -8,13 +14,9 @@ import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -55,6 +57,9 @@ public class RedactionMessageReceiver {
|
||||
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), analyzeRequest.getManualRedactions());
|
||||
log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result.getDuration());
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
|
||||
}
|
||||
|
||||
result.setMessageType(analyzeRequest.getMessageType());
|
||||
|
||||
+6
-6
@@ -643,7 +643,7 @@ public class Section {
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -801,7 +801,7 @@ public class Section {
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(cleanValue) && cleanValue.length() >= 3) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(cleanValue);
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(cleanValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -865,7 +865,7 @@ public class Section {
|
||||
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(match);
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -980,7 +980,7 @@ public class Section {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1018,7 +1018,7 @@ public class Section {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(value.trim());
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(value.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1046,7 +1046,7 @@ public class Section {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
localDictionaryAdds.computeIfAbsent(asType, (x) -> new HashSet<>()).add(line.trim());
|
||||
localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(line.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-11
@@ -221,20 +221,18 @@ public class EntityRedactionService {
|
||||
|
||||
private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {
|
||||
|
||||
analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> {
|
||||
analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key) == null) {
|
||||
log.warn("Dictionary {} is null", key);
|
||||
}
|
||||
if (dictionary.getLocalAccessMap().get(key) == null) {
|
||||
log.warn("Dictionary {} is null", key);
|
||||
}
|
||||
|
||||
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
|
||||
log.warn("Dictionary {} localEntries is null", key);
|
||||
}
|
||||
if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
|
||||
log.warn("Dictionary {} localEntries is null", key);
|
||||
}
|
||||
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
});
|
||||
});
|
||||
dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
|
||||
+36
-28
@@ -1,18 +1,28 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.experimental.UtilityClass;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@UtilityClass
|
||||
@SuppressWarnings("PMD")
|
||||
@@ -93,33 +103,31 @@ public class EntitySearchUtils {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {
|
||||
private void applyResizeRedactions(Set<Entity> entitiesWithPositions, ManualRedactions manualRedactions) {
|
||||
|
||||
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()){
|
||||
return entitiesWithPositions;
|
||||
if (manualRedactions == null || manualRedactions.getResizeRedactions() == null || manualRedactions.getResizeRedactions().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
entitiesWithPositions.forEach(e -> e.getPositionSequences().forEach(pos -> {
|
||||
manualRedactions.getResizeRedactions().stream().filter(resize -> resize.getStatus().equals(AnnotationStatus.APPROVED)).forEach(resize -> {
|
||||
if (resize.getAnnotationId().equals(pos.getId())) {
|
||||
if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
|
||||
int start = e.getWord().indexOf(resize.getValue());
|
||||
e.setStart(e.getStart() + start);
|
||||
e.setEnd(e.getStart() + resize.getValue().length());
|
||||
e.setResized(true);
|
||||
e.setWord(resize.getValue());
|
||||
} else if(resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())){
|
||||
int start = resize.getValue().indexOf(e.getWord());
|
||||
e.setStart(e.getStart() - start);
|
||||
e.setEnd(e.getStart() + resize.getValue().length());
|
||||
e.setResized(true);
|
||||
e.setWord(resize.getValue());
|
||||
entitiesWithPositions.forEach(e -> e.getPositionSequences()
|
||||
.forEach(pos -> manualRedactions.getResizeRedactions().stream().filter(resize -> resize.getStatus().equals(AnnotationStatus.APPROVED)).forEach(resize -> {
|
||||
if (resize.getAnnotationId().equals(pos.getId())) {
|
||||
if (resize.getValue().length() < e.getWord().length() && e.getWord().contains(resize.getValue())) {
|
||||
int start = e.getWord().indexOf(resize.getValue());
|
||||
e.setStart(e.getStart() + start);
|
||||
e.setEnd(e.getStart() + resize.getValue().length());
|
||||
e.setResized(true);
|
||||
e.setWord(resize.getValue());
|
||||
} else if (resize.getValue().length() > e.getWord().length() && resize.getValue().contains(e.getWord())) {
|
||||
int start = resize.getValue().indexOf(e.getWord());
|
||||
e.setStart(e.getStart() - start);
|
||||
e.setEnd(e.getStart() + resize.getValue().length());
|
||||
e.setResized(true);
|
||||
e.setWord(resize.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}));
|
||||
})));
|
||||
|
||||
return entitiesWithPositions;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+25
-23
@@ -1,27 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.SystemUtils;
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
@@ -35,6 +13,30 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.SystemUtils;
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@@ -72,7 +74,7 @@ public class PdfSegmentationService {
|
||||
Document document = new Document();
|
||||
List<Page> pages = new ArrayList<>();
|
||||
|
||||
pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(1024 * 1024 * 64));
|
||||
pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(67108864L));
|
||||
pdDocument.setAllSecurityToBeRemoved(true);
|
||||
long pageCount = pdDocument.getNumberOfPages();
|
||||
|
||||
|
||||
+6
-103
@@ -12,17 +12,14 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
@@ -32,10 +29,6 @@ import java.io.InputStream;
|
||||
public class RedactionStorageService {
|
||||
|
||||
private final StorageService storageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private final DslJson<Object> dslJson = new DslJson<>(Settings.basicSetup());
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public InputStream getStoredObject(String storageId) {
|
||||
@@ -54,156 +47,66 @@ public class RedactionStorageService {
|
||||
@SneakyThrows
|
||||
@Timed("redactmanager_storeObject")
|
||||
public void storeObject(String dossierId, String fileId, FileType fileType, Object any) {
|
||||
|
||||
try(var byteArrayOutputStream = serializeObject(any)){
|
||||
storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), byteArrayOutputStream);
|
||||
}
|
||||
storageService.storeJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), any);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Timed("redactmanager_serializeObject")
|
||||
private ByteArrayOutputStream serializeObject(Object any) {
|
||||
|
||||
try {
|
||||
var baos = new ByteArrayOutputStream();
|
||||
dslJson.serialize(any, baos);
|
||||
return baos;
|
||||
} catch (com.dslplatform.json.SerializationException e) {
|
||||
// Fails on file 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf
|
||||
var baos = new ByteArrayOutputStream();
|
||||
objectMapper.writeValue(baos, any);
|
||||
dslJson.newWriter();
|
||||
return baos;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getImportedRedactions")
|
||||
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Imported redactions not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeImportedRedactions(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeImportedRedactions")
|
||||
private ImportedRedactions deserializeImportedRedactions(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not imported redactions", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getRedactionLog")
|
||||
public RedactionLog getRedactionLog(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),RedactionLog.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeRedactionLog(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeRedactionLog")
|
||||
private RedactionLog deserializeRedactionLog(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert RedactionLog", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getText")
|
||||
public Text getText(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT),Text.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
log.debug("Text not available.");
|
||||
return null;
|
||||
}
|
||||
|
||||
return deserializeText(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeText")
|
||||
private Text deserializeText(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(Text.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert Text", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getNerEntities")
|
||||
public NerEntities getNerEntities(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("NER Entities are not available.");
|
||||
}
|
||||
|
||||
return deserializeNerEntities(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeNerEntities")
|
||||
private NerEntities deserializeNerEntities(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert NER Entities", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_getSectionGrid")
|
||||
public SectionGrid getSectionGrid(String dossierId, String fileId) {
|
||||
|
||||
InputStreamResource inputStreamResource;
|
||||
try {
|
||||
inputStreamResource = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID));
|
||||
return storageService.readJSONObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID),SectionGrid.class);
|
||||
} catch (StorageObjectDoesNotExist e) {
|
||||
throw new NotFoundException("Section Grid is not available.");
|
||||
}
|
||||
|
||||
return deserializeSectionGrid(inputStreamResource);
|
||||
}
|
||||
|
||||
|
||||
@Timed("redactmanager_deserializeSectionGrid")
|
||||
private SectionGrid deserializeSectionGrid(InputStreamResource inputStreamResource) {
|
||||
|
||||
try {
|
||||
return dslJson.deserialize(SectionGrid.class, inputStreamResource.getInputStream());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Could not convert SectionGrid", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
+36
-33
@@ -1,22 +1,25 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
|
||||
import com.iqser.red.storage.commons.service.S3StorageService;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
public class FileSystemBackedStorageService implements StorageService{
|
||||
public class FileSystemBackedStorageService implements StorageService {
|
||||
|
||||
private final Map<String, File> dataMap = new HashMap<>();
|
||||
|
||||
@@ -50,34 +53,47 @@ public class FileSystemBackedStorageService implements StorageService{
|
||||
|
||||
}
|
||||
|
||||
public List<String> listPaths(){
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> void storeJSONObject(String objectId, T any) {
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
getMapper().writeValue(new FileOutputStream(tempFile), any);
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
private ObjectMapper getMapper() {
|
||||
var objectMapper = new ObjectMapper();
|
||||
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
objectMapper.findAndRegisterModules();
|
||||
return objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public <T> T readJSONObject(String objectId, Class<T> clazz) {
|
||||
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
|
||||
throw new StorageObjectDoesNotExist("Stored object not found");
|
||||
}
|
||||
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
|
||||
}
|
||||
|
||||
public List<String> listPaths() {
|
||||
return new ArrayList<>(dataMap.keySet());
|
||||
}
|
||||
|
||||
|
||||
public List<String> listFilePaths(){
|
||||
public List<String> listFilePaths() {
|
||||
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void storeObject(String objectId, byte[] data) {
|
||||
File tempFile = File.createTempFile("storage", objectId.replace("/","-"));
|
||||
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
IOUtils.write(data, fileOutputStream);
|
||||
}
|
||||
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String objectId, InputStream stream) {
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
try (var fileOutputStream = new FileOutputStream(tempFile)) {
|
||||
IOUtils.copy(stream, fileOutputStream);
|
||||
}
|
||||
|
||||
@@ -85,19 +101,6 @@ public class FileSystemBackedStorageService implements StorageService{
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
@SneakyThrows
|
||||
public void storeObject(String objectId, ByteArrayOutputStream stream) {
|
||||
|
||||
File tempFile = File.createTempFile("test", ".tmp");
|
||||
try(var fileOutputStream = new FileOutputStream(tempFile)){
|
||||
stream.writeTo(fileOutputStream);
|
||||
}
|
||||
|
||||
dataMap.put(objectId, tempFile);
|
||||
}
|
||||
|
||||
|
||||
public void clearStorage() {
|
||||
this.dataMap.forEach((k, v) -> {
|
||||
v.delete();
|
||||
|
||||
+17
-24
@@ -30,10 +30,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
@@ -604,8 +601,7 @@ public class RedactionIntegrationTest {
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
@@ -897,9 +893,9 @@ public class RedactionIntegrationTest {
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.manualRedactions(manualRedactions)
|
||||
.colors(colors)
|
||||
.types(types)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
@@ -1627,8 +1623,7 @@ public class RedactionIntegrationTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@@ -1695,7 +1690,7 @@ public class RedactionIntegrationTest {
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), IOUtils.toByteArray(importedRedactions.getInputStream()));
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
@@ -1711,11 +1706,11 @@ public class RedactionIntegrationTest {
|
||||
if (entry.getValue() == null) {
|
||||
return;
|
||||
}
|
||||
if (entry.getValue().equals("David")){
|
||||
assertThat(entry.getImportedRedactionIntersections().size()).isEqualTo(1);
|
||||
if (entry.getValue().equals("David")) {
|
||||
assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
|
||||
}
|
||||
if (entry.getValue().equals("annotation")){
|
||||
assertThat(entry.getImportedRedactionIntersections().size()).isEqualTo(0);
|
||||
if (entry.getValue().equals("annotation")) {
|
||||
assertThat(entry.getImportedRedactionIntersections()).isEmpty();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1728,12 +1723,12 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void testExpandByPrefixRegEx() throws IOException {
|
||||
|
||||
assertThat(dictionary.get(AUTHOR).contains("Robinson"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mrs. Robinson"));
|
||||
assertThat(dictionary.get(AUTHOR).contains("Bojangles"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mr. Bojangles"));
|
||||
assertThat(dictionary.get(AUTHOR).contains("Tambourine Man"));
|
||||
assertThat(!dictionary.get(AUTHOR).contains("Mr. Tambourine Man"));
|
||||
assertThat(dictionary.get(AUTHOR)).contains("Robinson");
|
||||
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mrs. Robinson");
|
||||
assertThat(dictionary.get(AUTHOR)).contains("Bojangles");
|
||||
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Bojangles");
|
||||
assertThat(dictionary.get(AUTHOR)).contains("Tambourine Man");
|
||||
assertThat(dictionary.get(AUTHOR)).doesNotContain("Mr. Tambourine Man");
|
||||
|
||||
String fileName = "files/mr-mrs.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
@@ -1775,9 +1770,7 @@ public class RedactionIntegrationTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
|
||||
return request;
|
||||
|
||||
|
||||
+135
-82
@@ -10,12 +10,14 @@ import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.math.BigInteger;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.DirectoryStream;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
@@ -25,7 +27,6 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
@@ -48,6 +49,7 @@ import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.annotation.IfProfileValue;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
@@ -86,7 +88,7 @@ import lombok.SneakyThrows;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Import(RulesTest.RedactionIntegrationTestConfiguration.class)
|
||||
@Import(RulesTest.RulesTestConfiguration.class)
|
||||
public class RulesTest {
|
||||
|
||||
private static final String RULES_PATH = "drools/testRules.drl";
|
||||
@@ -111,7 +113,9 @@ public class RulesTest {
|
||||
private static final String IMPORTED_REDACTION = "imported_redaction";
|
||||
private static final String PII = "PII";
|
||||
private static final String RESOURCES_PATH = "src/test/resources/";
|
||||
private static final String REDACTION_LOG_PATH = RESOURCES_PATH + "RedactionLog/";
|
||||
|
||||
private static final String REDACTION_LOG_PATH = "RedactionLog/";
|
||||
private static final String REDACTION_LOG_RESOURCES_PATH = RESOURCES_PATH + REDACTION_LOG_PATH;
|
||||
private final static String TEST_DOSSIER_TEMPLATE_ID = "123";
|
||||
private final static String TEST_DOSSIER_ID = "123";
|
||||
private String TEST_FILE_ID = "123";
|
||||
@@ -151,34 +155,6 @@ public class RulesTest {
|
||||
private LegalBasisClient legalBasisClient;
|
||||
|
||||
|
||||
@After
|
||||
public void cleanupStorage() {
|
||||
|
||||
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Before
|
||||
public void stubClients() {
|
||||
|
||||
@@ -201,9 +177,7 @@ public class RulesTest {
|
||||
.hexColor("#ffe187")
|
||||
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
|
||||
.build()));
|
||||
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS)).rank(rankTypeMap.get(DOSSIER_REDACTIONS)).build()));
|
||||
|
||||
mockDictionaryCalls(null);
|
||||
mockDictionaryCalls(0L);
|
||||
@@ -212,8 +186,18 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
@After
|
||||
public void cleanupStorage() {
|
||||
|
||||
if (this.storageService instanceof FileSystemBackedStorageService) {
|
||||
((FileSystemBackedStorageService) this.storageService).clearStorage();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for given file and saves it here: REDACTION_LOG_PATH.
|
||||
* If the RedactionLog already exists, it will be overwritten
|
||||
* Test is ignored, because it's for manual tests.
|
||||
*/
|
||||
@Ignore
|
||||
@@ -221,25 +205,28 @@ public class RulesTest {
|
||||
public void generateRedactionLogForOneFile() {
|
||||
|
||||
String fileName = "files/Compounds/31 A14111B - EU AIR3 - MCP Section 1 - Identity of the plant protection product.pdf";
|
||||
generateAndSaveRedactionLog(fileName);
|
||||
generateAndSaveRedactionLog(fileName, true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Generates RedactionLog for all files and saves it here: REDACTION_LOG_PATH.
|
||||
* Test is ignored, because it's for manual tests.
|
||||
* If a RedactionLog already exists, the generating for this will be skipped
|
||||
* Please commit generated files, if not the following tests will fail
|
||||
*/
|
||||
@Ignore
|
||||
@Test
|
||||
@Ignore// TODO PSC REMOVE
|
||||
public void generateRedactionLogForAllFiles() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will generate RedactionLog for " + files.size() + " files.");
|
||||
TEST_FILE_ID = "1000";
|
||||
files.forEach(this::generateAndSaveRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Analyses file and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||
* If RedactionLog Json does not exist, test will fail.
|
||||
@@ -256,43 +243,62 @@ public class RulesTest {
|
||||
|
||||
/**
|
||||
* Analyses all files and compares its RedactionLog with saved one from here: REDACTION_LOG_PATH.
|
||||
* If RedactionLogs Json does not exist, test will fail.
|
||||
* Test is ignored, because it's for manual tests.
|
||||
* If RedactionLog Json for one file does not exist, whole test will fail.
|
||||
*/
|
||||
@Ignore
|
||||
@IfProfileValue(name = "test-groups", value = "rules-test")
|
||||
@Test
|
||||
public void analyseAllFilesAndCompareRedactionLogs() {
|
||||
|
||||
Set<String> files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(RESOURCES_PATH));
|
||||
System.out.println("Will analyse " + files.size() + " files and compare its RedactionLogs.");
|
||||
TEST_FILE_ID = "5000";
|
||||
|
||||
System.out.println("files: " + files);
|
||||
|
||||
files.forEach(this::analyseFileAndCompareRedactionLog);
|
||||
}
|
||||
|
||||
|
||||
@Test// TODO PSC REMOVE
|
||||
@Ignore
|
||||
public void foo() {
|
||||
String fileName = "oo-home/local-working-dir/3080193/RED-REDACTIONSERVICENIGHT0-JOB1/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Compounds/26 A8637C - EU AIR3 - LCP Section 10 - Ecotoxicological studies on the plant protection product - Reference list.pdf";
|
||||
loadSavedRedactionLog(fileName);
|
||||
}
|
||||
|
||||
|
||||
private void generateAndSaveRedactionLog(String fileName) {
|
||||
generateAndSaveRedactionLog(fileName, false);
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
public void generateAndSaveRedactionLog(String fileName) {
|
||||
private void generateAndSaveRedactionLog(String fileName, boolean force) {
|
||||
|
||||
increaseTestFileId();
|
||||
if (!force && doesRedactionLogExist(fileName)) {
|
||||
System.out.println("Skip generating RedactionLog as Json for " + fileName + " because it already exists.");
|
||||
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
} else {
|
||||
generateTestFileId(fileName);
|
||||
|
||||
loadNerForTest();
|
||||
System.out.println("Generate RedactionLog as Json for " + fileName + " with fileId " + TEST_FILE_ID);
|
||||
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
loadNerForTest();
|
||||
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
AnalyzeRequest request = prepareStorage(fileName);
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
saveRedactionLogAsJson(redactionLog, fileName);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void analyseFileAndCompareRedactionLog(String fileName) {
|
||||
|
||||
increaseTestFileId();
|
||||
generateTestFileId(fileName);
|
||||
System.out.println("Analyse " + fileName + " with fileId " + TEST_FILE_ID + " and compare it with its saved RedactionLog");
|
||||
|
||||
RedactionLog savedRedactionLog = loadSavedRedactionLog(fileName);
|
||||
@@ -410,19 +416,65 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
private void increaseTestFileId() {
|
||||
private boolean doesRedactionLogExist(String pdfFileName) {
|
||||
|
||||
TEST_FILE_ID = Integer.toString(Integer.parseInt(TEST_FILE_ID) + 1);
|
||||
File pdfFile = new File(pdfFileName);
|
||||
String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
return file.exists();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void generateTestFileId(String fileName) {
|
||||
|
||||
MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
|
||||
messageDigest.update(fileName.getBytes());
|
||||
TEST_FILE_ID = new BigInteger(1, messageDigest.digest()).toString(16);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
|
||||
String directory = REDACTION_LOG_RESOURCES_PATH + pdfFile.getParentFile().getPath();
|
||||
File dr = new File(directory);
|
||||
boolean created = dr.mkdirs();
|
||||
if (created) {
|
||||
System.out.println("Directory was created");
|
||||
}
|
||||
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
objectMapper.writeValue(file, redactionLog);
|
||||
|
||||
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private RedactionLog loadSavedRedactionLog(String pdfFileName) {
|
||||
|
||||
System.out.println("XXXXXXXXX pdfFileName: " + pdfFileName);
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
String directory = pdfFile.getParentFile().getPath();
|
||||
if (StringUtils.contains(directory, RESOURCES_PATH) || StringUtils.contains(directory, StringUtils.replace(RESOURCES_PATH, "/", "\\"))) {
|
||||
directory = directory + "/" + REDACTION_LOG_PATH;
|
||||
} else {
|
||||
directory = REDACTION_LOG_RESOURCES_PATH + directory;
|
||||
}
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
System.out.println("XXXXXXXXX directory: " + directory);
|
||||
System.out.println("XXXXXXXXX fileName: " + fileName);
|
||||
|
||||
assertThat(file).exists();
|
||||
|
||||
return objectMapper.readValue(file, RedactionLog.class);
|
||||
@@ -433,8 +485,7 @@ public class RulesTest {
|
||||
private void loadNerForTest() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
|
||||
var bytes = IOUtils.toByteArray(responseJson.getInputStream());
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
|
||||
}
|
||||
|
||||
|
||||
@@ -457,36 +508,13 @@ public class RulesTest {
|
||||
.lastProcessed(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var bytes = IOUtils.toByteArray(stream);
|
||||
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), bytes);
|
||||
storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), stream);
|
||||
|
||||
return request;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private void saveRedactionLogAsJson(RedactionLog redactionLog, String pdfFileName) {
|
||||
|
||||
File pdfFile = new File(pdfFileName);
|
||||
|
||||
String directory = REDACTION_LOG_PATH + pdfFile.getParentFile().getPath();
|
||||
File dr = new File(directory);
|
||||
boolean created = dr.mkdirs();
|
||||
if (created) {
|
||||
System.out.println("Directory was created");
|
||||
}
|
||||
|
||||
String fileName = StringUtils.replace(pdfFile.getName(), ".pdf", ".json");
|
||||
File file = new File(directory, fileName);
|
||||
|
||||
objectMapper.writeValue(file, redactionLog);
|
||||
|
||||
System.out.println("Saved RedactionLog for " + fileName + " here " + directory);
|
||||
}
|
||||
|
||||
|
||||
private void loadDictionaryForTest() {
|
||||
|
||||
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
|
||||
@@ -600,8 +628,14 @@ public class RulesTest {
|
||||
getFileNames(fileNames, path);
|
||||
} else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
|
||||
String absolutePath = path.toAbsolutePath().toString();
|
||||
System.out.println("absolutePath: " + absolutePath);
|
||||
int pos = StringUtils.indexOf(absolutePath, StringUtils.replace(RESOURCES_PATH, "/", "\\")) + 18;
|
||||
|
||||
pos = StringUtils.indexOf(absolutePath, RESOURCES_PATH) + 18;
|
||||
|
||||
System.out.println("pos: " + pos);
|
||||
fileNames.add(StringUtils.substring(absolutePath, pos));
|
||||
System.out.println("Added: " + StringUtils.substring(absolutePath, pos));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -633,9 +667,28 @@ public class RulesTest {
|
||||
}
|
||||
|
||||
|
||||
private static String loadFromClassPath(String path) {
|
||||
|
||||
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
|
||||
if (resource == null) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
|
||||
}
|
||||
try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String str;
|
||||
while ((str = br.readLine()) != null) {
|
||||
sb.append(str).append("\n");
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("could not load classpath resource: " + path, e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
|
||||
public static class RedactionIntegrationTestConfiguration {
|
||||
public static class RulesTestConfiguration {
|
||||
|
||||
@Bean
|
||||
public KieContainer kieContainer() {
|
||||
|
||||
+45
-32
@@ -1,22 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.realdata;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.nullable;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.*;
|
||||
import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
@@ -35,13 +30,27 @@ import org.springframework.core.io.Resource;
|
||||
import org.springframework.core.io.support.ResourcePatternResolver;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.when;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@@ -114,9 +123,8 @@ public class LiveDataIntegrationTest {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
var jsonNode = objectMapper.readTree(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "types.json").getInputStream());
|
||||
types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(
|
||||
new TypeReference<List<Type>>() {
|
||||
}));
|
||||
types = objectMapper.treeToValue(jsonNode.get("types"), objectMapper.getTypeFactory().constructType(new TypeReference<List<Type>>() {
|
||||
}));
|
||||
|
||||
types.forEach(t -> {
|
||||
t.setId(t.getType());
|
||||
@@ -126,27 +134,28 @@ public class LiveDataIntegrationTest {
|
||||
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
|
||||
when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
|
||||
|
||||
|
||||
when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(), Colors.class));
|
||||
|
||||
when(dictionaryClient.getDictionaryForType(anyString(), nullable(Long.class))).then(answer -> {
|
||||
String typeName = answer.getArgument(0);
|
||||
|
||||
var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
|
||||
if(found.isPresent()) {
|
||||
if (found.isPresent()) {
|
||||
var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
|
||||
type.setEntries(getEntries(typeName, type.getTypeId()));
|
||||
|
||||
return type;
|
||||
}else{
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
dictionaryService.updateDictionary("dossierTemplateId","dossierId");
|
||||
dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
|
||||
}
|
||||
|
||||
|
||||
public void simulateIncrement(List<String> values, String deltaTypeName, long version) {
|
||||
|
||||
when(dictionaryClient.getVersion(anyString())).thenReturn(version);
|
||||
when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(version);
|
||||
|
||||
@@ -154,7 +163,7 @@ public class LiveDataIntegrationTest {
|
||||
String typeName = answer.getArgument(0);
|
||||
|
||||
var found = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst();
|
||||
if(found.isPresent()) {
|
||||
if (found.isPresent()) {
|
||||
|
||||
var type = types.stream().filter(t -> t.getType().equalsIgnoreCase(typeName)).findFirst().get();
|
||||
|
||||
@@ -165,7 +174,7 @@ public class LiveDataIntegrationTest {
|
||||
}
|
||||
|
||||
return type;
|
||||
}else{
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
@@ -174,17 +183,20 @@ public class LiveDataIntegrationTest {
|
||||
|
||||
@Test
|
||||
public void testUpdateDictionary() {
|
||||
|
||||
dictionaryService.updateDictionary("dossierTemplateId", "dossierId");
|
||||
|
||||
var dict = dictionaryService.getDeepCopyDictionary("dossierTemplateId", "dossierId");
|
||||
assertThat(dict.getLocalAccessMap().size()).isEqualTo(12);
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
private List<DictionaryEntry> getEntries(String typeName, String typeId) {
|
||||
|
||||
Resource[] dictionaryResources = resourcePatternResolver.getResources("classpath:" + BASE_DIR + EFSA_SANITISATION_GFL_V1 + "dictionaries/**");
|
||||
for (var resource : dictionaryResources) {
|
||||
if (resource.getFilename().contains(typeName)) {
|
||||
if (Objects.requireNonNull(resource.getFilename()).contains(typeName)) {
|
||||
|
||||
List<String> lines = IOUtils.readLines(resource.getInputStream());
|
||||
return lines.stream().map(l -> new DictionaryEntry(0, l, 0L, false, typeId)).collect(Collectors.toList());
|
||||
@@ -193,4 +205,5 @@ public class LiveDataIntegrationTest {
|
||||
}
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+33
-36
@@ -1,23 +1,17 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
import org.junit.Ignore;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.platform.commons.util.StringUtils;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
@@ -31,20 +25,20 @@ import org.springframework.context.annotation.Import;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils.getTemporaryDirectory;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@@ -89,15 +83,18 @@ public class PdfSegmentationServiceTest {
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testMapping(){
|
||||
public void testMapping() {
|
||||
|
||||
ClassPathResource responseJson = new ClassPathResource("files/image_response.json");
|
||||
ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);
|
||||
|
||||
Map<Integer, List<PdfImage>> images = new HashMap<>();
|
||||
imageServiceResponse.getData().stream().forEach(imageMetadata -> {
|
||||
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber()));
|
||||
});
|
||||
imageServiceResponse.getData()
|
||||
.forEach(imageMetadata -> images.computeIfAbsent(imageMetadata.getPosition().getPageNumber(), x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry()
|
||||
.getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification()
|
||||
.getLabel()
|
||||
.toUpperCase(Locale.ROOT)), imageMetadata.isAlpha(), imageMetadata.getPosition().getPageNumber())));
|
||||
|
||||
System.out.println("object");
|
||||
}
|
||||
|
||||
+10153
File diff suppressed because it is too large
Load Diff
+533
@@ -0,0 +1,533 @@
|
||||
{
|
||||
"analysisVersion": 1,
|
||||
"analysisNumber": 0,
|
||||
"redactionLogEntry": [
|
||||
{
|
||||
"id": "96c1362cb4ea87461d8f5f141df1912a",
|
||||
"type": "PII",
|
||||
"value": "patrick.gardinal@syngenta.com",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 368.44
|
||||
},
|
||||
"width": 141.39014,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 10,
|
||||
"textBefore": "60 51 E-mail: ",
|
||||
"textAfter": null,
|
||||
"comments": [],
|
||||
"startOffset": 295,
|
||||
"endOffset": 324,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "74b528293158d5266ace1eaf9ee2ad80",
|
||||
"type": "PII",
|
||||
"value": "+41 (0) 61 323 60 51",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "Producer of the active substance: Cyprodinil",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 169.96002
|
||||
},
|
||||
"width": 94.37213,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 12,
|
||||
"textBefore": "Gardinal Telephone number: ",
|
||||
"textAfter": " E-mail: patrick.gardinal@syngenta.com",
|
||||
"comments": [],
|
||||
"startOffset": 188,
|
||||
"endOffset": 208,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "d282cd8f02e2628c14342ed52d69c517",
|
||||
"type": "PII",
|
||||
"value": "+41 (61) 323 6152",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.1 Applicant",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 610.12
|
||||
},
|
||||
"width": 83.43994,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 9,
|
||||
"textBefore": "Micheletti Telephone number: ",
|
||||
"textAfter": " E-mail: sebastien.micheletti@syngenta.com",
|
||||
"comments": [],
|
||||
"startOffset": 164,
|
||||
"endOffset": 181,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "245cfd060d4fb078bb211fabfce6d470",
|
||||
"type": "PII",
|
||||
"value": "patrick.gardinal@syngenta.com",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "Producer of the active substance: Cyprodinil",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 144.64001
|
||||
},
|
||||
"width": 141.39014,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 12,
|
||||
"textBefore": "60 51 E-mail: ",
|
||||
"textAfter": null,
|
||||
"comments": [],
|
||||
"startOffset": 217,
|
||||
"endOffset": 246,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "67dc43606b8e1489a0594e5b779fa428",
|
||||
"type": "PII",
|
||||
"value": "Patrick Gardinal",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 419.08
|
||||
},
|
||||
"width": 73.440186,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 10,
|
||||
"textBefore": "Basel; Switzerland Contact: ",
|
||||
"textAfter": " Telephone number:",
|
||||
"comments": [],
|
||||
"startOffset": 231,
|
||||
"endOffset": 247,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "de79428457282a0781a827b322b5732a",
|
||||
"type": "PII",
|
||||
"value": "sebastien.micheletti@syngenta.com",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.1 Applicant",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 584.8
|
||||
},
|
||||
"width": 161.04922,
|
||||
"height": 10.98,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 9,
|
||||
"textBefore": "323 6152 E-mail: ",
|
||||
"textAfter": null,
|
||||
"comments": [],
|
||||
"startOffset": 190,
|
||||
"endOffset": 223,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "41340ab190d66ca6dce0ba965ce95f59",
|
||||
"type": "PII",
|
||||
"value": "Patrick Gardinal",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "Producer of the active substance: Cyprodinil",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 195.28003
|
||||
},
|
||||
"width": 73.440186,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 12,
|
||||
"textBefore": "Basel; Switzerland Contact: ",
|
||||
"textAfter": " Telephone number:",
|
||||
"comments": [],
|
||||
"startOffset": 153,
|
||||
"endOffset": 169,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "02dd2efb100839e705cdcf6f3141d9d5",
|
||||
"type": "PII",
|
||||
"value": "Sebastien Micheletti",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.1 Applicant",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 634.84
|
||||
},
|
||||
"width": 91.21478,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 9,
|
||||
"textBefore": "Basel; Switzerland Contact: ",
|
||||
"textAfter": " Telephone number:",
|
||||
"comments": [],
|
||||
"startOffset": 125,
|
||||
"endOffset": 145,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
},
|
||||
{
|
||||
"id": "530c4f47f0afc48432bd849a90918f41",
|
||||
"type": "PII",
|
||||
"value": "+41 (0) 61 323 60 51",
|
||||
"reason": "Personal information found",
|
||||
"matchedRule": 23,
|
||||
"rectangle": false,
|
||||
"legalBasis": "Article 39(e)(3) of Regulation (EC) No 178/2002",
|
||||
"imported": false,
|
||||
"redacted": true,
|
||||
"section": "CP 1.2 Producer of the Plant Protection Product and the Active\nSubstances",
|
||||
"color": [
|
||||
0.4,
|
||||
0.8,
|
||||
1.0
|
||||
],
|
||||
"positions": [
|
||||
{
|
||||
"topLeft": {
|
||||
"x": 286.92,
|
||||
"y": 393.76
|
||||
},
|
||||
"width": 94.37213,
|
||||
"height": 11.52,
|
||||
"page": 5
|
||||
}
|
||||
],
|
||||
"sectionNumber": 10,
|
||||
"textBefore": "Gardinal Telephone number: ",
|
||||
"textAfter": " E-mail: patrick.gardinal@syngenta.com",
|
||||
"comments": [],
|
||||
"startOffset": 266,
|
||||
"endOffset": 286,
|
||||
"imageHasTransparency": false,
|
||||
"excluded": false,
|
||||
"sourceId": null,
|
||||
"changes": [
|
||||
{
|
||||
"analysisNumber": 0,
|
||||
"type": "ADDED",
|
||||
"dateTime": "2022-07-25T11:12:04.3835183+02:00"
|
||||
}
|
||||
],
|
||||
"manualChanges": [],
|
||||
"engines": [
|
||||
"RULE"
|
||||
],
|
||||
"reference": [],
|
||||
"importedRedactionIntersections": [],
|
||||
"dossierDictionaryEntry": false,
|
||||
"localManualRedaction": false,
|
||||
"dictionaryEntry": false,
|
||||
"image": false,
|
||||
"falsePositive": false,
|
||||
"manuallyRemoved": false,
|
||||
"hint": false,
|
||||
"recommendation": false
|
||||
}
|
||||
],
|
||||
"legalBasis": [],
|
||||
"dictionaryVersion": 0,
|
||||
"dossierDictionaryVersion": 0,
|
||||
"rulesVersion": 0,
|
||||
"legalBasisVersion": 0
|
||||
}
|
||||
+40982
File diff suppressed because it is too large
Load Diff
+4377
File diff suppressed because it is too large
Load Diff
+6927
File diff suppressed because it is too large
Load Diff
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"12ac3d26ce82a77e469c64d56081662d","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Table in: COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":214.8,"y":639.64},"width":55.039078,"height":11.52,"page":4}],"sectionNumber":3,"textBefore":"Report: Document F/02, ","textAfter":". (2014). Cyprodinil","comments":[],"startOffset":23,"endOffset":35,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dd614353319e7c5a8cf8b3346dc12fac","type":"CBI_author","value":"Micheletti S","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"COPY OF THE APPLICATION FOR RENEWAL, MADE UNDER COMMISSION","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":306.84,"y":667.12},"width":60.71997,"height":12.0,"page":4}],"sectionNumber":5,"textBefore":"F report (","textAfter":"., 2014).","comments":[],"startOffset":247,"endOffset":259,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398559.316208600}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"36a5e8909a18a5080dd423afb8efd7a7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":159.0,"y":320.0},"width":10.208038,"height":-21.999977,"page":1}],"sectionNumber":3,"textBefore":"Product Metabolism and ","textAfter":" 1 of","comments":[],"startOffset":368,"endOffset":372,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"dca119c3aadd348bc43c0b700c0075c3","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":100.0,"y":81.0},"width":10.208031,"height":-22.0,"page":1}],"sectionNumber":3,"textBefore":"Pages RAM 465/02 ","textAfter":" 1 Tel:","comments":[],"startOffset":399,"endOffset":403,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"ef6e0f0ed413dbd7299d992d9a50840b","type":"CBI_author","value":"Crook","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":374.0,"y":389.0},"width":10.515198,"height":-27.02501,"page":1}],"sectionNumber":5,"textBefore":"Author : SJ ","textAfter":" Analytical Science","comments":[],"startOffset":12,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"0fac36714dd9d1c745565c6dfe06b9b7","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":514.14215,"y":47.94165},"width":24.39972,"height":11.591161,"page":2}],"sectionNumber":4,"textBefore":"RAM 465/02 ","textAfter":" 2 Summary","comments":[],"startOffset":11,"endOffset":15,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398772.538085900}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"71770acab1fa63f46c026966890b5231","type":"vertebrate","value":"rabbit","reason":null,"matchedRule":0,"rectangle":false,"legalBasis":null,"imported":false,"redacted":false,"section":"","color":[1.0,0.52156866,0.96862745],"positions":[{"topLeft":{"x":77.73999,"y":537.62},"width":-11.043678,"height":36.630165,"page":1}],"sectionNumber":4,"textBefore":null,"textAfter":null,"comments":[],"startOffset":41,"endOffset":47,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398329.875800000}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":true,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"64eb5ff4ea1db8665fb6e57c5369b411","type":"PII","value":"Page","reason":"Personal information found","matchedRule":19,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Text in table","color":[0.4,0.8,1.0],"positions":[{"topLeft":{"x":154.48782,"y":654.18},"width":9.992157,"height":19.882175,"page":1}],"sectionNumber":8,"textBefore":null,"textAfter":" 1 of","comments":[],"startOffset":54,"endOffset":58,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398731.072100800}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[{"id":"9f6837120752b85c20e27d4cb199bd51","type":"CBI_author","value":"B. Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":712.96},"width":31.589584,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":null,"textAfter":" F. Bar","comments":[],"startOffset":0,"endOffset":6,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"fb930b7f3acf65a74b0b59fee32cc6fa","type":"CBI_author","value":"F. Bar","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":125.92424,"y":712.96},"width":29.137009,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"B. Foo ","textAfter":" Foo, B.","comments":[],"startOffset":7,"endOffset":13,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false},{"id":"435224747b87ed87b60ab363e6610fba","type":"CBI_author","value":"Foo","reason":"Author found","matchedRule":1,"rectangle":false,"legalBasis":"Article 39(e)(3) of Regulation (EC) No 178/2002","imported":false,"redacted":true,"section":"Header","color":[1.0,0.88235295,0.5294118],"positions":[{"topLeft":{"x":92.520004,"y":686.8},"width":18.338272,"height":12.231649,"page":1}],"sectionNumber":3,"textBefore":"Foo F. Bar ","textAfter":", B.","comments":[],"startOffset":14,"endOffset":17,"imageHasTransparency":false,"excluded":false,"sourceId":null,"changes":[{"analysisNumber":0,"type":"ADDED","dateTime":1658398909.230209400}],"manualChanges":[],"engines":["DICTIONARY"],"reference":[],"importedRedactionIntersections":[],"dossierDictionaryEntry":false,"recommendation":false,"hint":false,"dictionaryEntry":true,"falsePositive":false,"image":false,"localManualRedaction":false,"manuallyRemoved":false}],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
{"analysisVersion":1,"analysisNumber":0,"redactionLogEntry":[],"legalBasis":[],"dictionaryVersion":0,"dossierDictionaryVersion":0,"rulesVersion":0,"legalBasisVersion":0}
|
||||
+1
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user