Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Kilian Schuettler
|
593e3121b9 | ||
|
Kilian Schuettler
|
5718db1725 | ||
|
Kilian Schuettler
|
89f2ed7655 | ||
|
Kilian Schuettler
|
daf221a52f | ||
|
Kilian Schuettler
|
afc051592b | ||
|
Kilian Schuettler
|
140c29ddc7 | ||
|
Kilian Schuettler
|
af1f4193d0 |
@@ -69,7 +69,3 @@ build/
|
||||
**/.DS_Store
|
||||
**/classpath-data.json
|
||||
**/dependencies-and-licenses-overview.txt
|
||||
gradle.properties
|
||||
gradlew
|
||||
gradlew.bat
|
||||
gradle/
|
||||
|
||||
@@ -4,33 +4,3 @@ include:
|
||||
- project: 'gitlab/gitlab'
|
||||
ref: 'main'
|
||||
file: 'ci-templates/gradle_java.yml'
|
||||
|
||||
deploy:
|
||||
stage: deploy
|
||||
tags:
|
||||
- dind
|
||||
script:
|
||||
- echo "Building with gradle version ${BUILDVERSION}"
|
||||
- gradle -Pversion=${BUILDVERSION} publish
|
||||
- gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
|
||||
- echo "BUILDVERSION=$BUILDVERSION" >> version.env
|
||||
artifacts:
|
||||
reports:
|
||||
dotenv: version.env
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
- if: $CI_COMMIT_BRANCH =~ /^release/
|
||||
- if: $CI_COMMIT_TAG
|
||||
|
||||
pages:
|
||||
stage: build
|
||||
tags:
|
||||
- dind
|
||||
script:
|
||||
- echo "Generating Javadoc..."
|
||||
- gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
|
||||
- mkdir public
|
||||
- mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
|
||||
artifacts:
|
||||
paths:
|
||||
- public
|
||||
|
||||
@@ -6,15 +6,11 @@ plugins {
|
||||
jacoco
|
||||
}
|
||||
|
||||
group = "com.iqser.red.service"
|
||||
group = "com.iqser.red"
|
||||
|
||||
java.sourceCompatibility = JavaVersion.VERSION_17
|
||||
java.targetCompatibility = JavaVersion.VERSION_17
|
||||
|
||||
pmd {
|
||||
isConsoleOutput = true
|
||||
}
|
||||
|
||||
tasks.pmdMain {
|
||||
pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
|
||||
}
|
||||
@@ -43,6 +39,25 @@ tasks.jacocoTestReport {
|
||||
}
|
||||
}
|
||||
|
||||
allprojects {
|
||||
publishing {
|
||||
publications {
|
||||
create<MavenPublication>(name) {
|
||||
from(components["java"])
|
||||
}
|
||||
}
|
||||
repositories {
|
||||
maven {
|
||||
url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
|
||||
credentials {
|
||||
username = providers.gradleProperty("mavenUser").getOrNull();
|
||||
password = providers.gradleProperty("mavenPassword").getOrNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
java {
|
||||
withJavadocJar()
|
||||
}
|
||||
@@ -57,4 +72,4 @@ repositories {
|
||||
password = providers.gradleProperty("mavenPassword").getOrNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-3
@@ -9,12 +9,13 @@
|
||||
</description>
|
||||
|
||||
<rule ref="category/java/errorprone.xml">
|
||||
<exclude name="DataflowAnomalyAnalysis"/>
|
||||
<exclude name="MissingSerialVersionUID"/>
|
||||
<exclude name="NullAssignment"/>
|
||||
<exclude name="AvoidLiteralsInIfCondition"/>
|
||||
<exclude name="AvoidDuplicateLiterals"/>
|
||||
<exclude name="NullAssignment"/>
|
||||
<exclude name="AvoidFieldNameMatchingMethodName"/>
|
||||
<exclude name="AssignmentInOperand"/>
|
||||
<exclude name="BeanMembersShouldSerialize"/>
|
||||
</rule>
|
||||
|
||||
</ruleset>
|
||||
</ruleset>
|
||||
@@ -10,13 +10,15 @@
|
||||
|
||||
|
||||
<rule ref="category/java/errorprone.xml">
|
||||
<exclude name="DataflowAnomalyAnalysis"/>
|
||||
<exclude name="MissingSerialVersionUID"/>
|
||||
<exclude name="NullAssignment"/>
|
||||
<exclude name="AvoidLiteralsInIfCondition"/>
|
||||
<exclude name="AvoidDuplicateLiterals"/>
|
||||
<exclude name="NullAssignment"/>
|
||||
<exclude name="AvoidFieldNameMatchingMethodName"/>
|
||||
<exclude name="AvoidFieldNameMatchingTypeName"/>
|
||||
<exclude name="AssignmentInOperand"/>
|
||||
<exclude name="TestClassWithoutTestCases"/>
|
||||
<exclude name="BeanMembersShouldSerialize"/>
|
||||
</rule>
|
||||
|
||||
</ruleset>
|
||||
</ruleset>
|
||||
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
dir=${PWD##*/}
|
||||
gradle assemble
|
||||
|
||||
# Get the current Git branch
|
||||
branch=$(git rev-parse --abbrev-ref HEAD)
|
||||
|
||||
# Get the short commit hash (first 5 characters)
|
||||
commit_hash=$(git rev-parse --short=5 HEAD)
|
||||
|
||||
# Combine branch and commit hash
|
||||
buildName="${USER}-${branch}-${commit_hash}"
|
||||
|
||||
gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
|
||||
echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"
|
||||
@@ -1,28 +1,14 @@
|
||||
|
||||
plugins {
|
||||
id("com.iqser.red.service.java-conventions")
|
||||
id("io.freefair.lombok") version "8.4"
|
||||
id("io.freefair.lombok") version "8.1.0"
|
||||
}
|
||||
|
||||
description = "redaction-service-api-v1"
|
||||
|
||||
dependencies {
|
||||
implementation("org.springframework:spring-web:6.0.12")
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
|
||||
implementation("org.springframework:spring-web:6.0.6")
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725")
|
||||
}
|
||||
|
||||
publishing {
|
||||
publications {
|
||||
create<MavenPublication>(name) {
|
||||
from(components["java"])
|
||||
}
|
||||
}
|
||||
repositories {
|
||||
maven {
|
||||
url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
|
||||
credentials {
|
||||
username = providers.gradleProperty("mavenUser").getOrNull();
|
||||
password = providers.gradleProperty("mavenPassword").getOrNull();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class AnalyzeResponse {
|
||||
|
||||
private String fileId;
|
||||
private List<UnprocessedManualEntity> unprocessedManualEntities;
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class DroolsSyntaxDeprecatedWarnings {
|
||||
|
||||
Integer line;
|
||||
Integer column;
|
||||
String message;
|
||||
}
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class DroolsSyntaxErrorMessage {
|
||||
|
||||
Integer line;
|
||||
Integer column;
|
||||
String message;
|
||||
}
|
||||
-36
@@ -1,36 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class DroolsSyntaxValidation {
|
||||
|
||||
@Builder.Default
|
||||
List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
|
||||
@Builder.Default
|
||||
List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
|
||||
|
||||
|
||||
public void addErrorMessage(int line, int column, String message) {
|
||||
|
||||
getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build());
|
||||
}
|
||||
|
||||
public boolean isCompiled() {
|
||||
|
||||
return droolsSyntaxErrorMessages.isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class MigrationRequest {
|
||||
|
||||
String dossierTemplateId;
|
||||
String dossierId;
|
||||
String fileId;
|
||||
ManualRedactions manualRedactions;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class MigrationResponse {
|
||||
|
||||
String dossierId;
|
||||
String fileId;
|
||||
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public class QueueNames {
|
||||
|
||||
public static final String REDACTION_QUEUE = "redactionQueue";
|
||||
public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
|
||||
public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
|
||||
public static final String REDACTION_DQL = "redactionDQL";
|
||||
|
||||
public static final String MIGRATION_QUEUE = "migrationQueue";
|
||||
public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
|
||||
public static final String MIGRATION_DLQ = "migrationDLQ";
|
||||
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class RuleValidationModel {
|
||||
|
||||
String ruleFileType;
|
||||
String rulesString;
|
||||
|
||||
}
|
||||
-27
@@ -1,27 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@Builder
|
||||
@EqualsAndHashCode
|
||||
public class UnprocessedManualEntity {
|
||||
|
||||
private String annotationId;
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
private String section;
|
||||
@Builder.Default
|
||||
private List<Position> positions = new ArrayList<>();
|
||||
}
|
||||
+1
-4
@@ -4,12 +4,9 @@ import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
|
||||
import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
|
||||
|
||||
public interface RedactionResource {
|
||||
|
||||
@PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel);
|
||||
void testRules(@RequestBody String rules);
|
||||
|
||||
}
|
||||
|
||||
@@ -3,93 +3,58 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
|
||||
plugins {
|
||||
application
|
||||
id("com.iqser.red.service.java-conventions")
|
||||
id("org.springframework.boot") version "3.1.5"
|
||||
id("io.spring.dependency-management") version "1.1.3"
|
||||
id("org.sonarqube") version "4.4.1.3373"
|
||||
id("io.freefair.lombok") version "8.4"
|
||||
id("org.springframework.boot") version "3.0.6"
|
||||
id("io.spring.dependency-management") version "1.1.0"
|
||||
id("org.sonarqube") version "4.2.1.3168"
|
||||
id("io.freefair.lombok") version "8.1.0"
|
||||
}
|
||||
|
||||
description = "redaction-service-server-v1"
|
||||
|
||||
|
||||
val layoutParserVersion = "0.94.0"
|
||||
val jacksonVersion = "2.15.2"
|
||||
val droolsVersion = "9.44.0.Final"
|
||||
val pdfBoxVersion = "3.0.0"
|
||||
val persistenceServiceVersion = "2.360.0"
|
||||
val springBootStarterVersion = "3.1.5"
|
||||
|
||||
configurations {
|
||||
all {
|
||||
exclude(group = "commons-logging", module = "commons-logging")
|
||||
exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
|
||||
exclude(group = "com.iqser.red.commons", module = "logging-commons")
|
||||
}
|
||||
}
|
||||
extra["layoutparserVersion"] = "0.8.0"
|
||||
|
||||
dependencies {
|
||||
implementation(project(":redaction-service-api-v1"))
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725")
|
||||
implementation("com.knecon.fforesight:layoutparser-service-internal-api:0.12.0")
|
||||
|
||||
implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
|
||||
implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
|
||||
implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
|
||||
implementation("com.iqser.red.commons:spring-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") }
|
||||
implementation("com.iqser.red.commons:metric-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") }
|
||||
|
||||
implementation("com.iqser.red.commons:spring-commons:6.2.0")
|
||||
implementation("com.iqser.red.commons:metric-commons:2.3.0")
|
||||
implementation("com.iqser.red.commons:dictionary-merge-commons:1.3.0")
|
||||
implementation("com.iqser.red.commons:storage-commons:2.1.0")
|
||||
implementation("com.knecon.fforesight:tenant-commons:0.10.0")
|
||||
|
||||
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
|
||||
implementation("com.iqser.red.commons:storage-commons:2.45.0")
|
||||
implementation("com.knecon.fforesight:tenant-commons:0.21.0")
|
||||
implementation("com.knecon.fforesight:tracing-commons:0.5.0")
|
||||
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
|
||||
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-aop:3.0.4")
|
||||
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:2.14.2")
|
||||
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2")
|
||||
implementation("org.ahocorasick:ahocorasick:0.6.3")
|
||||
implementation("org.javassist:javassist:3.29.2-GA")
|
||||
|
||||
implementation("org.drools:drools-engine:${droolsVersion}")
|
||||
implementation("org.drools:drools-mvel:${droolsVersion}")
|
||||
implementation("org.kie:kie-spring:7.74.1.Final")
|
||||
implementation("org.drools:drools-engine:8.37.0.Final")
|
||||
implementation("org.drools:drools-mvel:8.37.0.Final")
|
||||
implementation("org.kie:kie-spring:7.73.0.Final")
|
||||
|
||||
implementation("org.locationtech.jts:jts-core:1.19.0")
|
||||
|
||||
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
|
||||
implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
|
||||
implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
|
||||
testImplementation("org.apache.pdfbox:pdfbox:3.0.0-alpha2")
|
||||
testImplementation("org.apache.pdfbox:pdfbox-tools:3.0.0-alpha2")
|
||||
|
||||
implementation("net.logstash.logback:logstash-logback-encoder:7.4")
|
||||
implementation("ch.qos.logback:logback-classic")
|
||||
|
||||
implementation("org.reflections:reflections:0.10.2")
|
||||
|
||||
testImplementation(project(":rules-management"))
|
||||
testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
|
||||
testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
|
||||
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
|
||||
testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
|
||||
testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
|
||||
exclude(
|
||||
group = "com.iqser.red.service",
|
||||
module = "persistence-service-shared-api-v1"
|
||||
)
|
||||
}
|
||||
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1")
|
||||
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.4")
|
||||
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
|
||||
testImplementation("com.knecon.fforesight:layoutparser-service-processor:0.12.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") }
|
||||
}
|
||||
|
||||
tasks.test {
|
||||
configure<JacocoTaskExtension> {
|
||||
excludes = listOf("org/drools/**/*")
|
||||
}
|
||||
minHeapSize = "512m"
|
||||
maxHeapSize = "1024m"
|
||||
}
|
||||
|
||||
|
||||
tasks.named<BootBuildImage>("bootBuildImage") {
|
||||
|
||||
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
|
||||
environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8")
|
||||
|
||||
imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
|
||||
imageName.set("nexus.knecon.com:5001/ff/${project.name}:${project.version}")
|
||||
if (project.hasProperty("buildbootDockerHostNetwork")) {
|
||||
network.set("host")
|
||||
}
|
||||
@@ -105,45 +70,5 @@ tasks.named<BootBuildImage>("bootBuildImage") {
|
||||
email.set(providers.gradleProperty("mavenEmail").getOrNull())
|
||||
url.set("https://nexus.knecon.com:5001/")
|
||||
}
|
||||
|
||||
val dockerTag = "nexus.knecon.com:5001/red/${project.name}:${project.version}"
|
||||
tags.set(listOf(dockerTag))
|
||||
}
|
||||
}
|
||||
|
||||
fun parseDroolsImports(droolsFilePath: String): List<String> {
|
||||
val imports = mutableListOf<String>()
|
||||
val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
|
||||
val desiredPrefix = "com.iqser.red.service.redaction.v1"
|
||||
|
||||
File(droolsFilePath).forEachLine { line ->
|
||||
importPattern.find(line)?.let { matchResult ->
|
||||
val importPath = matchResult.groupValues[1].trim()
|
||||
if (importPath.startsWith(desiredPrefix)) {
|
||||
val formattedPath = importPath.replace('.', '/')
|
||||
imports.add("$formattedPath.java")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return imports
|
||||
}
|
||||
|
||||
val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
|
||||
|
||||
tasks.register("generateJavaDoc", Javadoc::class) {
|
||||
dependsOn("compileJava")
|
||||
dependsOn("delombok")
|
||||
classpath = project.sourceSets["main"].runtimeClasspath
|
||||
source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
|
||||
include(droolsImports)
|
||||
}
|
||||
destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
|
||||
|
||||
options.memberLevel = JavadocMemberLevel.PUBLIC
|
||||
(options as StandardJavadocDocletOptions).apply {
|
||||
header = "Redaction Service ${project.version}"
|
||||
footer = "Redaction Service ${project.version}"
|
||||
title = "API Documentation for Redaction Service ${project.version}"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-16
@@ -6,22 +6,19 @@ import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.cloud.openfeign.EnableFeignClients;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Import;
|
||||
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.storage.commons.StorageAutoConfiguration;
|
||||
import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
|
||||
|
||||
import io.micrometer.core.aop.TimedAspect;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
import io.micrometer.observation.ObservationRegistry;
|
||||
import io.micrometer.observation.aop.ObservedAspect;
|
||||
|
||||
@EnableCaching
|
||||
@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
|
||||
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
|
||||
@EnableFeignClients(basePackageClasses = RulesClient.class)
|
||||
@@ -35,10 +32,6 @@ public class Application {
|
||||
SpringApplication.run(Application.class, args);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
|
||||
return new ObservedAspect(observationRegistry);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public TimedAspect timedAspect(MeterRegistry registry) {
|
||||
@@ -46,18 +39,10 @@ public class Application {
|
||||
return new TimedAspect(registry);
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public DictionaryMergeService dictionaryMergeService() {
|
||||
|
||||
return new DictionaryMergeService();
|
||||
}
|
||||
|
||||
|
||||
@Bean(initMethod = "findDeprecatedMethods")
|
||||
public DeprecatedElementsFinder deprecatedElementsFinder() {
|
||||
|
||||
return new DeprecatedElementsFinder();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-108
@@ -1,108 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import static java.util.stream.Collectors.groupingBy;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.lang.reflect.Type;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.reflections.Reflections;
|
||||
import org.reflections.scanners.Scanners;
|
||||
import org.reflections.util.ConfigurationBuilder;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class DeprecatedElementsFinder {
|
||||
|
||||
public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
|
||||
private Set<Method> deprecatedMethods;
|
||||
@Getter
|
||||
private Map<String, String> deprecatedMethodsSignaturesMap;
|
||||
private Set<Class<?>> deprecatedClasses;
|
||||
@Getter
|
||||
private SearchImplementation classesSearchImplementation;
|
||||
@Getter
|
||||
private SearchImplementation methodsSearchImplementation;
|
||||
|
||||
|
||||
public void findDeprecatedMethods() {
|
||||
|
||||
deprecatedMethods = new HashSet<>();
|
||||
deprecatedClasses = new HashSet<>();
|
||||
deprecatedMethodsSignaturesMap = new HashMap<>();
|
||||
|
||||
Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
|
||||
.setExpandSuperTypes(true)
|
||||
.setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
|
||||
|
||||
deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
|
||||
|
||||
deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
|
||||
|
||||
if (!deprecatedClasses.isEmpty()) {
|
||||
deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
|
||||
.collect(Collectors.toSet())));
|
||||
classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
|
||||
.map(Class::getName)
|
||||
.collect(Collectors.toSet()), false);
|
||||
}
|
||||
|
||||
if (!deprecatedMethods.isEmpty()) {
|
||||
methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
|
||||
.map(Method::getName)
|
||||
.collect(Collectors.toSet()), false);
|
||||
deprecatedMethodsSignaturesMap = new HashMap<>();
|
||||
Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
|
||||
.collect(groupingBy(Method::getName));
|
||||
for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
|
||||
deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
|
||||
key -> deprecatedMethodsNameMap.get(deprecatedMethod)
|
||||
.stream()
|
||||
.map(this::getMethodSignature)
|
||||
.collect(Collectors.joining("\n")));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
|
||||
}
|
||||
|
||||
|
||||
public Set<Method> getDeprecatedMethods() {
|
||||
|
||||
if (this.deprecatedMethods == null) {
|
||||
findDeprecatedMethods();
|
||||
}
|
||||
return this.deprecatedMethods;
|
||||
}
|
||||
|
||||
|
||||
public Set<Class<?>> getDeprecatedClasses() {
|
||||
|
||||
if (this.deprecatedClasses == null) {
|
||||
findDeprecatedMethods();
|
||||
}
|
||||
return this.deprecatedClasses;
|
||||
}
|
||||
|
||||
private String getMethodSignature(Method method) {
|
||||
|
||||
String methodName = method.getName();
|
||||
String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
|
||||
.map(Type::getTypeName)
|
||||
.collect(Collectors.joining(", "));
|
||||
|
||||
return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
|
||||
}
|
||||
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.springframework.boot.autoconfigure.cache.RedisCacheManagerBuilderCustomizer;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.redis.cache.RedisCacheConfiguration;
|
||||
import org.springframework.data.redis.connection.RedisConnectionFactory;
|
||||
import org.springframework.data.redis.core.RedisTemplate;
|
||||
|
||||
@Configuration
|
||||
public class RedisCachingConfiguration {
|
||||
|
||||
@Bean
|
||||
public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() {
|
||||
|
||||
return (builder) -> builder.withCacheConfiguration("documentDataCache",
|
||||
RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
|
||||
|
||||
RedisTemplate<String, Object> template = new RedisTemplate<>();
|
||||
template.setConnectionFactory(connectionFactory);
|
||||
return template;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import com.iqser.red.commons.spring.ErrorMessage;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
+5
-7
@@ -3,11 +3,9 @@ package com.iqser.red.service.redaction.v1.server.controller;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
|
||||
import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
|
||||
import com.iqser.red.service.redaction.v1.resources.RedactionResource;
|
||||
import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -17,14 +15,14 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@RequiredArgsConstructor
|
||||
public class RedactionController implements RedactionResource {
|
||||
|
||||
private final DroolsSyntaxValidationService droolsSyntaxValidationService;
|
||||
private final DroolsExecutionService droolsExecutionService;
|
||||
|
||||
|
||||
@Override
|
||||
public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) {
|
||||
public void testRules(@RequestBody String rules) {
|
||||
|
||||
try {
|
||||
return droolsSyntaxValidationService.testRules(rulesValidationModel);
|
||||
droolsExecutionService.testRules(rules);
|
||||
} catch (Exception e) {
|
||||
throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
+4
-8
@@ -1,11 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.controller;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
|
||||
import com.iqser.red.service.redaction.v1.resources.RuleBuilderResource;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.RuleBuilderModelService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -13,16 +12,13 @@ import lombok.RequiredArgsConstructor;
|
||||
@RequiredArgsConstructor
|
||||
public class RuleBuilderController implements RuleBuilderResource {
|
||||
|
||||
private final RuleBuilderModelService ruleBuilderModelService;
|
||||
|
||||
|
||||
@Override
|
||||
public RuleBuilderModel getRuleBuilderModel() {
|
||||
|
||||
RuleBuilderModel ruleBuilderModel = new RuleBuilderModel();
|
||||
|
||||
ruleBuilderModel.setWhenClauses(Collections.emptyList());
|
||||
ruleBuilderModel.setThenConditions(Collections.emptyList());
|
||||
|
||||
return ruleBuilderModel;
|
||||
return ruleBuilderModelService.getRuleBuilderModel();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-7
@@ -1,6 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document;
|
||||
|
||||
import java.io.Serializable;
|
||||
package com.iqser.red.service.redaction.v1.server.document.data;
|
||||
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
|
||||
@@ -11,15 +9,13 @@ import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class DocumentData implements Serializable {
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class DocumentData {
|
||||
|
||||
DocumentPage[] documentPages;
|
||||
DocumentTextData[] documentTextData;
|
||||
+17
-19
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service.document;
|
||||
package com.iqser.red.service.redaction.v1.server.document.data.mapper;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
@@ -6,24 +6,23 @@ import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Headline;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
|
||||
@@ -77,7 +76,6 @@ public class DocumentGraphMapper {
|
||||
node.setLeafTextBlock(textBlock);
|
||||
}
|
||||
List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
|
||||
entryData.getEngines().forEach(engine -> node.addEngine(engine));
|
||||
node.setTreeId(treeId);
|
||||
|
||||
switch (entryData.getType()) {
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.data.mapper;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class PropertiesMapper {
|
||||
|
||||
String imageType = "imageType";
|
||||
private final String transparency = "transparency";
|
||||
private final String position = "position";
|
||||
String id = "id";
|
||||
String row = "row";
|
||||
String col = "col";
|
||||
String header = "header";
|
||||
String bBox = "bBox";
|
||||
String numberOfRows = "numberOfRows";
|
||||
String numberOfCols = "numberOfCols";
|
||||
|
||||
|
||||
public Map<String, String> buildImageProperties(Image image) {
|
||||
|
||||
Map<String, String> properties = new HashMap<>();
|
||||
properties.put(imageType, image.getImageType().toString());
|
||||
properties.put(transparency, String.valueOf(image.isTransparent()));
|
||||
properties.put(position, toString(image.getPosition()));
|
||||
properties.put(id, image.getId());
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
||||
private String toString(Rectangle2D rectangle2D) {
|
||||
|
||||
return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
|
||||
}
|
||||
|
||||
|
||||
public Map<String, String> buildTableCellProperties(TableCell tableCell) {
|
||||
|
||||
Map<String, String> properties = new HashMap<>();
|
||||
properties.put(row, String.valueOf(tableCell.getRow()));
|
||||
properties.put(col, String.valueOf(tableCell.getCol()));
|
||||
properties.put(header, String.valueOf(tableCell.isHeader()));
|
||||
|
||||
if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
|
||||
throw new IllegalArgumentException("TableCell can only occur on a single page!");
|
||||
}
|
||||
String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
|
||||
properties.put(bBox, bBoxString);
|
||||
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
||||
public Map<String, String> buildTableProperties(Table table) {
|
||||
|
||||
Map<String, String> properties = new HashMap<>();
|
||||
properties.put(numberOfRows, String.valueOf(table.getNumberOfRows()));
|
||||
properties.put(numberOfCols, String.valueOf(table.getNumberOfCols()));
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
||||
public void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
|
||||
|
||||
builder.imageType(ImageType.fromString(properties.get(imageType)));
|
||||
builder.transparent(Boolean.parseBoolean(properties.get(transparency)));
|
||||
builder.position(parseRectangle2D(properties.get(position)));
|
||||
builder.id(properties.get(id));
|
||||
}
|
||||
|
||||
|
||||
public void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
|
||||
|
||||
builder.row(Integer.parseInt(properties.get(row)));
|
||||
builder.col(Integer.parseInt(properties.get(col)));
|
||||
builder.header(Boolean.parseBoolean(properties.get(header)));
|
||||
builder.bBox(parseRectangle2D(properties.get(bBox)));
|
||||
}
|
||||
|
||||
|
||||
public void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
|
||||
|
||||
builder.numberOfRows(Integer.parseInt(properties.get(numberOfRows)));
|
||||
builder.numberOfCols(Integer.parseInt(properties.get(numberOfCols)));
|
||||
}
|
||||
|
||||
|
||||
private Rectangle2D parseRectangle2D(String bBox) {
|
||||
|
||||
List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
|
||||
return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
|
||||
}
|
||||
|
||||
}
|
||||
+23
-24
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@@ -6,21 +6,20 @@ import java.util.Collection;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Setter;
|
||||
|
||||
@Setter
|
||||
@EqualsAndHashCode
|
||||
@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
|
||||
public class TextRange implements Comparable<TextRange> {
|
||||
public class Boundary implements Comparable<Boundary> {
|
||||
|
||||
private int start;
|
||||
private int end;
|
||||
|
||||
|
||||
public TextRange(int start, int end) {
|
||||
public Boundary(int start, int end) {
|
||||
|
||||
if (start > end) {
|
||||
throw new IllegalArgumentException(format("start: %d > end: %d", start, end));
|
||||
@@ -48,15 +47,15 @@ public class TextRange implements Comparable<TextRange> {
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(TextRange textRange) {
|
||||
public boolean contains(Boundary boundary) {
|
||||
|
||||
return start <= textRange.start() && textRange.end() <= end;
|
||||
return start <= boundary.start() && boundary.end() <= end;
|
||||
}
|
||||
|
||||
|
||||
public boolean containedBy(TextRange textRange) {
|
||||
public boolean containedBy(Boundary boundary) {
|
||||
|
||||
return textRange.contains(this);
|
||||
return boundary.contains(this);
|
||||
}
|
||||
|
||||
|
||||
@@ -84,18 +83,18 @@ public class TextRange implements Comparable<TextRange> {
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(TextRange textRange) {
|
||||
public boolean intersects(Boundary boundary) {
|
||||
|
||||
return textRange.start() < this.end && this.start < textRange.end();
|
||||
return boundary.start() < this.end && this.start < boundary.end();
|
||||
}
|
||||
|
||||
|
||||
public List<TextRange> split(List<Integer> splitIndices) {
|
||||
public List<Boundary> split(List<Integer> splitIndices) {
|
||||
|
||||
if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
|
||||
throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
|
||||
}
|
||||
List<TextRange> splitBoundaries = new LinkedList<>();
|
||||
List<Boundary> splitBoundaries = new LinkedList<>();
|
||||
int previousIndex = start;
|
||||
for (int splitIndex : splitIndices) {
|
||||
|
||||
@@ -103,19 +102,19 @@ public class TextRange implements Comparable<TextRange> {
|
||||
if (splitIndex == previousIndex) {
|
||||
continue;
|
||||
}
|
||||
splitBoundaries.add(new TextRange(previousIndex, splitIndex));
|
||||
splitBoundaries.add(new Boundary(previousIndex, splitIndex));
|
||||
previousIndex = splitIndex;
|
||||
}
|
||||
splitBoundaries.add(new TextRange(previousIndex, end));
|
||||
splitBoundaries.add(new Boundary(previousIndex, end));
|
||||
return splitBoundaries;
|
||||
}
|
||||
|
||||
|
||||
public static TextRange merge(Collection<TextRange> boundaries) {
|
||||
public static Boundary merge(Collection<Boundary> boundaries) {
|
||||
|
||||
int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new);
|
||||
int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new);
|
||||
return new TextRange(minStart, maxEnd);
|
||||
int minStart = boundaries.stream().mapToInt(Boundary::start).min().orElseThrow(IllegalArgumentException::new);
|
||||
int maxEnd = boundaries.stream().mapToInt(Boundary::end).max().orElseThrow(IllegalArgumentException::new);
|
||||
return new Boundary(minStart, maxEnd);
|
||||
}
|
||||
|
||||
|
||||
@@ -127,12 +126,12 @@ public class TextRange implements Comparable<TextRange> {
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(TextRange textRange) {
|
||||
public int compareTo(Boundary boundary) {
|
||||
|
||||
if (end < textRange.end() && start < textRange.start()) {
|
||||
if (end < boundary.end() && start < boundary.start()) {
|
||||
return -1;
|
||||
}
|
||||
if (start > textRange.start() && end > textRange.end()) {
|
||||
if (start > boundary.start() && end > boundary.end()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -146,7 +145,7 @@ public class TextRange implements Comparable<TextRange> {
|
||||
* @param textBlock TextBlock to check whitespaces against
|
||||
* @return trimmed boundary
|
||||
*/
|
||||
public TextRange trim(TextBlock textBlock) {
|
||||
public Boundary trim(TextBlock textBlock) {
|
||||
|
||||
if (this.length() == 0) {
|
||||
return this;
|
||||
@@ -161,7 +160,7 @@ public class TextRange implements Comparable<TextRange> {
|
||||
trimmedEnd--;
|
||||
}
|
||||
|
||||
return new TextRange(trimmedStart, Math.max(trimmedEnd, trimmedStart));
|
||||
return new Boundary(trimmedStart, Math.max(trimmedEnd, trimmedStart));
|
||||
}
|
||||
|
||||
}
|
||||
+11
-11
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
@@ -11,17 +11,17 @@ import java.util.stream.Collector;
|
||||
|
||||
import com.google.common.base.Functions;
|
||||
|
||||
public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<TextRange>, List<TextRange>> {
|
||||
public class ConsecutiveBoundaryCollector implements Collector<Boundary, List<Boundary>, List<Boundary>> {
|
||||
|
||||
@Override
|
||||
public Supplier<List<TextRange>> supplier() {
|
||||
public Supplier<List<Boundary>> supplier() {
|
||||
|
||||
return LinkedList::new;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public BiConsumer<List<TextRange>, TextRange> accumulator() {
|
||||
public BiConsumer<List<Boundary>, Boundary> accumulator() {
|
||||
|
||||
return (existingList, boundary) -> {
|
||||
if (existingList.isEmpty()) {
|
||||
@@ -29,14 +29,14 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
|
||||
return;
|
||||
}
|
||||
|
||||
TextRange prevTextRange = existingList.get(existingList.size() - 1);
|
||||
if (prevTextRange.end() > boundary.start()) {
|
||||
throw new IllegalArgumentException(String.format("Can't concatenate %s and %s. Boundaries must be ordered!", prevTextRange, boundary));
|
||||
Boundary prevBoundary = existingList.get(existingList.size() - 1);
|
||||
if (prevBoundary.end() > boundary.start()) {
|
||||
throw new IllegalArgumentException(String.format("Can't concatenate %s and %s. Boundaries must be ordered!", prevBoundary, boundary));
|
||||
}
|
||||
|
||||
if (prevTextRange.end() == boundary.start()) {
|
||||
if (prevBoundary.end() == boundary.start()) {
|
||||
existingList.remove(existingList.size() - 1);
|
||||
existingList.add(TextRange.merge(List.of(prevTextRange, boundary)));
|
||||
existingList.add(Boundary.merge(List.of(prevBoundary, boundary)));
|
||||
} else {
|
||||
existingList.add(boundary);
|
||||
}
|
||||
@@ -45,7 +45,7 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
|
||||
|
||||
|
||||
@Override
|
||||
public BinaryOperator<List<TextRange>> combiner() {
|
||||
public BinaryOperator<List<Boundary>> combiner() {
|
||||
|
||||
return (list1, list2) -> {
|
||||
list1.addAll(list2);
|
||||
@@ -55,7 +55,7 @@ public class ConsecutiveBoundaryCollector implements Collector<TextRange, List<T
|
||||
|
||||
|
||||
@Override
|
||||
public Function<List<TextRange>, List<TextRange>> finisher() {
|
||||
public Function<List<Boundary>, List<Boundary>> finisher() {
|
||||
|
||||
return Functions.identity();
|
||||
}
|
||||
+9
-52
@@ -1,21 +1,20 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.GenericSemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -136,48 +135,6 @@ public class DocumentTree {
|
||||
}
|
||||
|
||||
|
||||
public Optional<SemanticNode> getNextSibling(List<Integer> treeId) {
|
||||
|
||||
var siblingTreeId = getNextSiblingId(treeId);
|
||||
if (!entryExists(siblingTreeId)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(getEntryById(siblingTreeId).getNode());
|
||||
}
|
||||
|
||||
|
||||
public List<Integer> getNextSiblingId(List<Integer> treeId) {
|
||||
|
||||
List<Integer> siblingTreeId = new LinkedList<>();
|
||||
for (int i = 0; i < treeId.size() - 1; i++) {
|
||||
siblingTreeId.add(treeId.get(i));
|
||||
}
|
||||
siblingTreeId.add(treeId.get(treeId.size() - 1) + 1);
|
||||
return siblingTreeId;
|
||||
}
|
||||
|
||||
|
||||
public Optional<SemanticNode> getPreviousSibling(List<Integer> treeId) {
|
||||
|
||||
var siblingTreeId = getPreviousSiblingId(treeId);
|
||||
if (!entryExists(siblingTreeId)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(getEntryById(siblingTreeId).getNode());
|
||||
}
|
||||
|
||||
|
||||
public List<Integer> getPreviousSiblingId(List<Integer> treeId) {
|
||||
|
||||
List<Integer> siblingTreeId = new LinkedList<>();
|
||||
for (int i = 0; i < treeId.size() - 1; i++) {
|
||||
siblingTreeId.add(treeId.get(i));
|
||||
}
|
||||
siblingTreeId.add(treeId.get(treeId.size() - 1) - 1);
|
||||
return siblingTreeId;
|
||||
}
|
||||
|
||||
|
||||
public Entry getEntryById(List<Integer> treeId) {
|
||||
|
||||
if (treeId.isEmpty()) {
|
||||
+1
-2
@@ -1,8 +1,7 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
public enum EntityType {
|
||||
ENTITY,
|
||||
HINT,
|
||||
RECOMMENDATION,
|
||||
FALSE_POSITIVE,
|
||||
FALSE_RECOMMENDATION
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public final class MatchedRule implements Comparable<MatchedRule> {
|
||||
|
||||
@Builder.Default
|
||||
RuleIdentifier ruleIdentifier = RuleIdentifier.empty();
|
||||
@Builder.Default
|
||||
String reason = "";
|
||||
@Builder.Default
|
||||
String legalBasis = "";
|
||||
boolean applied;
|
||||
boolean writeValueWithLineBreaks;
|
||||
@Builder.Default
|
||||
Set<RedactionEntity> references = Collections.emptySet();
|
||||
|
||||
|
||||
public static MatchedRule empty() {
|
||||
|
||||
return MatchedRule.builder().build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(MatchedRule matchedRule) {
|
||||
|
||||
RuleIdentifier otherRuleIdentifier = matchedRule.getRuleIdentifier();
|
||||
if (!Objects.equals(ruleIdentifier.type(), otherRuleIdentifier.type())) {
|
||||
if (Objects.equals(otherRuleIdentifier.type(), "MAN")) {
|
||||
return 1;
|
||||
}
|
||||
if (Objects.equals(ruleIdentifier.type(), "MAN")) {
|
||||
return -1;
|
||||
}
|
||||
if (Objects.equals(otherRuleIdentifier.type(), "X")) {
|
||||
return 1;
|
||||
}
|
||||
if (Objects.equals(ruleIdentifier.type(), "X")) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) {
|
||||
return otherRuleIdentifier.unit() - ruleIdentifier.unit();
|
||||
}
|
||||
return otherRuleIdentifier.id() - ruleIdentifier.id();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return "MatchedRule[" + "ruleIdentifier=" + ruleIdentifier + ", " + "reason=" + reason + ", " + "legalBasis=" + legalBasis + ", " + "applied=" + applied + ", " + "writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", " + "references=" + references + ']';
|
||||
}
|
||||
|
||||
}
|
||||
+41
-95
@@ -1,111 +1,64 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
public interface IEntity {
|
||||
public interface MatchedRuleHolder {
|
||||
|
||||
PriorityQueue<MatchedRule> getMatchedRuleList();
|
||||
|
||||
|
||||
ManualChangeOverwrite getManualOverwrite();
|
||||
boolean isIgnored();
|
||||
|
||||
|
||||
String getValue();
|
||||
boolean isRemoved();
|
||||
|
||||
|
||||
TextRange getTextRange();
|
||||
void setIgnored(boolean ignored);
|
||||
|
||||
|
||||
String type();
|
||||
void setRemoved(boolean ignored);
|
||||
|
||||
|
||||
default int length() {
|
||||
default boolean isApplied() {
|
||||
|
||||
return value().length();
|
||||
return getMatchedRule().isApplied();
|
||||
}
|
||||
|
||||
|
||||
default String value() {
|
||||
|
||||
return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
|
||||
}
|
||||
|
||||
|
||||
// Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
|
||||
default boolean applied() {
|
||||
|
||||
return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied());
|
||||
}
|
||||
|
||||
|
||||
default boolean skipped() {
|
||||
|
||||
return !applied();
|
||||
}
|
||||
|
||||
|
||||
default boolean ignored() {
|
||||
|
||||
return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored());
|
||||
}
|
||||
|
||||
|
||||
default boolean removed() {
|
||||
|
||||
return getManualOverwrite().getRemoved().orElse(getMatchedRule().isRemoved());
|
||||
}
|
||||
|
||||
|
||||
default boolean resized() {
|
||||
|
||||
return getManualOverwrite().getResized().orElse(false);
|
||||
}
|
||||
|
||||
|
||||
default boolean active() {
|
||||
|
||||
return !(removed() || ignored());
|
||||
}
|
||||
|
||||
|
||||
default boolean hasManualChanges() {
|
||||
|
||||
return !getManualOverwrite().getManualChangeLog().isEmpty();
|
||||
}
|
||||
|
||||
|
||||
default Set<TextEntity> references() {
|
||||
default Set<RedactionEntity> getReferences() {
|
||||
|
||||
return getMatchedRule().getReferences();
|
||||
}
|
||||
|
||||
|
||||
default void redact(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
|
||||
default boolean isActive() {
|
||||
|
||||
return !(isRemoved() || isIgnored());
|
||||
}
|
||||
|
||||
|
||||
default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
|
||||
|
||||
if (legalBasis.isBlank() || legalBasis.isEmpty()) {
|
||||
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
|
||||
}
|
||||
apply(ruleIdentifier, reason, legalBasis);
|
||||
}
|
||||
|
||||
|
||||
default void apply(@NonNull String ruleIdentifier, String reason, String legalBasis) {
|
||||
|
||||
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
|
||||
}
|
||||
|
||||
|
||||
default void apply(@NonNull String ruleIdentifier, String reason) {
|
||||
default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) {
|
||||
|
||||
apply(ruleIdentifier, reason, "n-a");
|
||||
addMatchedRule(MatchedRule.builder()
|
||||
.ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
|
||||
.reason(reason)
|
||||
.legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis))
|
||||
.applied(true)
|
||||
.build());
|
||||
}
|
||||
|
||||
|
||||
@@ -117,13 +70,27 @@ public interface IEntity {
|
||||
|
||||
default void remove(String ruleIdentifier, String reason) {
|
||||
|
||||
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).removed(true).build());
|
||||
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
|
||||
setRemoved(true);
|
||||
}
|
||||
|
||||
|
||||
default void ignore(String ruleIdentifier, String reason) {
|
||||
|
||||
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).ignored(true).build());
|
||||
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
|
||||
setIgnored(true);
|
||||
}
|
||||
|
||||
|
||||
private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) {
|
||||
|
||||
if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) {
|
||||
if (getMatchedRule() == null || !getMatchedRule().isApplied()) {
|
||||
return "n-a";
|
||||
}
|
||||
return getMatchedRule().getLegalBasis();
|
||||
}
|
||||
return legalBasis;
|
||||
}
|
||||
|
||||
|
||||
@@ -142,7 +109,7 @@ public interface IEntity {
|
||||
}
|
||||
|
||||
|
||||
default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection<TextEntity> references) {
|
||||
default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection<RedactionEntity> references) {
|
||||
|
||||
if (legalBasis.isBlank() || legalBasis.isEmpty()) {
|
||||
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
|
||||
@@ -157,7 +124,7 @@ public interface IEntity {
|
||||
}
|
||||
|
||||
|
||||
default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<TextEntity> references) {
|
||||
default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<RedactionEntity> references) {
|
||||
|
||||
getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build());
|
||||
}
|
||||
@@ -171,9 +138,6 @@ public interface IEntity {
|
||||
|
||||
default void addMatchedRules(Collection<MatchedRule> matchedRules) {
|
||||
|
||||
if (getMatchedRuleList().equals(matchedRules)) {
|
||||
return;
|
||||
}
|
||||
getMatchedRuleList().addAll(matchedRules);
|
||||
}
|
||||
|
||||
@@ -192,22 +156,4 @@ public interface IEntity {
|
||||
return getMatchedRuleList().peek();
|
||||
}
|
||||
|
||||
|
||||
default String buildReasonWithManualChangeDescriptions() {
|
||||
|
||||
if (getManualOverwrite().getDescriptions().isEmpty()) {
|
||||
return getMatchedRule().getReason();
|
||||
}
|
||||
if (getMatchedRule().getReason().isEmpty()) {
|
||||
return String.join(", ", getManualOverwrite().getDescriptions());
|
||||
}
|
||||
return getMatchedRule().getReason() + ", " + String.join(", ", getManualOverwrite().getDescriptions());
|
||||
}
|
||||
|
||||
|
||||
default String legalBasis() {
|
||||
|
||||
return getManualOverwrite().getLegalBasis().orElse(getMatchedRule().getLegalBasis());
|
||||
}
|
||||
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class RedactionEntity implements MatchedRuleHolder {
|
||||
|
||||
// initial values
|
||||
@EqualsAndHashCode.Include
|
||||
final Boundary boundary;
|
||||
@EqualsAndHashCode.Include
|
||||
final String type;
|
||||
@EqualsAndHashCode.Include
|
||||
final EntityType entityType;
|
||||
|
||||
// empty defaults
|
||||
boolean removed;
|
||||
boolean ignored;
|
||||
|
||||
boolean resized;
|
||||
boolean skipRemoveEntitiesContainedInLarger;
|
||||
boolean dictionaryEntry;
|
||||
boolean dossierDictionaryEntry;
|
||||
@Builder.Default
|
||||
Set<Engine> engines = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
|
||||
|
||||
// inferred on graph insertion
|
||||
String value;
|
||||
String textBefore;
|
||||
String textAfter;
|
||||
@Builder.Default
|
||||
Set<Page> pages = new HashSet<>();
|
||||
List<RedactionPosition> redactionPositionsPerPage;
|
||||
@Builder.Default
|
||||
List<SemanticNode> intersectingNodes = new LinkedList<>();
|
||||
SemanticNode deepestFullyContainingNode;
|
||||
|
||||
|
||||
public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) {
|
||||
|
||||
return RedactionEntity.builder().type(type).entityType(entityType).boundary(boundary).build();
|
||||
}
|
||||
|
||||
|
||||
public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
|
||||
|
||||
return intersectingNodes.stream().anyMatch(clazz::isInstance);
|
||||
}
|
||||
|
||||
|
||||
public boolean occursInNode(SemanticNode semanticNode) {
|
||||
|
||||
return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode));
|
||||
}
|
||||
|
||||
|
||||
public boolean isType(String type) {
|
||||
|
||||
return this.type.equals(type);
|
||||
}
|
||||
|
||||
|
||||
public boolean isAnyType(List<String> types) {
|
||||
|
||||
return types.contains(type);
|
||||
}
|
||||
|
||||
|
||||
public void addIntersectingNode(SemanticNode containingNode) {
|
||||
|
||||
intersectingNodes.add(containingNode);
|
||||
}
|
||||
|
||||
|
||||
public String getValueWithLineBreaks() {
|
||||
|
||||
return getDeepestFullyContainingNode().getTextBlock().subSequenceWithLineBreaks(getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public void removeFromGraph() {
|
||||
|
||||
intersectingNodes.forEach(node -> node.getEntities().remove(this));
|
||||
pages.forEach(page -> page.getEntities().remove(this));
|
||||
intersectingNodes = new LinkedList<>();
|
||||
deepestFullyContainingNode = null;
|
||||
pages = new HashSet<>();
|
||||
removed = true;
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionPosition> getRedactionPositionsPerPage() {
|
||||
|
||||
if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {
|
||||
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(boundary);
|
||||
|
||||
Page firstPage = rectanglesPerLinePerPage.keySet()
|
||||
.stream()
|
||||
.min(Comparator.comparingInt(Page::getNumber))
|
||||
.orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
|
||||
|
||||
String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
|
||||
redactionPositionsPerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildRedactionPosition(firstPage, id, entry)).toList();
|
||||
}
|
||||
return redactionPositionsPerPage;
|
||||
}
|
||||
|
||||
|
||||
private static RedactionPosition buildRedactionPosition(Page firstPage, String id, Map.Entry<Page, List<Rectangle2D>> entry) {
|
||||
|
||||
if (entry.getKey().equals(firstPage)) {
|
||||
return new RedactionPosition(id, entry.getKey(), entry.getValue());
|
||||
} else {
|
||||
return new RedactionPosition(id + "-" + entry.getKey().getNumber(), entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean containedBy(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.containedBy(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.contains(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(RedactionEntity redactionEntity) {
|
||||
|
||||
return this.boundary.intersects(redactionEntity.getBoundary());
|
||||
}
|
||||
|
||||
|
||||
public void addEngine(Engine engine) {
|
||||
|
||||
engines.add(engine);
|
||||
}
|
||||
|
||||
|
||||
public void addEngines(Set<Engine> engines) {
|
||||
|
||||
this.engines.addAll(engines);
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesAnnotationId(String manualRedactionId) {
|
||||
|
||||
return getRedactionPositionsPerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("Entity[\"");
|
||||
sb.append(value);
|
||||
sb.append("\", ");
|
||||
sb.append(boundary);
|
||||
sb.append(", pages[");
|
||||
pages.forEach(page -> {
|
||||
sb.append(page.getNumber());
|
||||
sb.append(", ");
|
||||
});
|
||||
sb.delete(sb.length() - 2, sb.length());
|
||||
sb.append("], type = \"");
|
||||
sb.append(type);
|
||||
sb.append("\", EntityType.");
|
||||
sb.append(entityType);
|
||||
sb.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
+5
-6
@@ -1,9 +1,9 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -13,13 +13,12 @@ import lombok.experimental.FieldDefaults;
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class PositionOnPage {
|
||||
public class RedactionPosition {
|
||||
|
||||
// Each entry in this list corresponds to an entry in the redaction log, this means:
|
||||
// A single entity might be represented by multiple redaction log entries
|
||||
// This is due to the RedactionLog only being able to handle a single page per entry.
|
||||
final String id;
|
||||
Page page;
|
||||
// Each entry in this list corresponds to an entry in the redaction log, this means:
|
||||
// An entity might be represented by multiple redaction log entries
|
||||
List<Rectangle2D> rectanglePerLine;
|
||||
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.entity;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record RuleIdentifier(String type, Integer unit, Integer id) {
|
||||
|
||||
public static RuleIdentifier fromString(String identifier) {
|
||||
|
||||
String[] values = identifier.split("\\.");
|
||||
if (values.length != 3) {
|
||||
throw new IllegalArgumentException("Illegal rule identifier provided: " + identifier);
|
||||
}
|
||||
String type = values[0];
|
||||
Integer group = Integer.parseInt(values[1]);
|
||||
Integer id = Integer.parseInt(values[2]);
|
||||
return new RuleIdentifier(type, group, id);
|
||||
}
|
||||
|
||||
|
||||
public static RuleIdentifier empty() {
|
||||
|
||||
return new RuleIdentifier("", null, null);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(type());
|
||||
if (Objects.nonNull(unit()) && Objects.nonNull(id())) {
|
||||
sb.append(".").append(unit()).append(".").append(id());
|
||||
} else if (Objects.nonNull(id())) {
|
||||
sb.append(".*.").append(id());
|
||||
} else if (Objects.nonNull(unit())) {
|
||||
sb.append(".").append(unit()).append(".*");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
+13
-14
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
@@ -10,17 +10,15 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@@ -29,20 +27,14 @@ import lombok.experimental.FieldDefaults;
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Document implements GenericSemanticNode {
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId = Collections.emptyList();
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
|
||||
Set<Page> pages;
|
||||
DocumentTree documentTree;
|
||||
Integer numberOfPages;
|
||||
TextBlock textBlock;
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
@Builder.Default
|
||||
static final SectionIdentifier sectionIdentifier = SectionIdentifier.document();
|
||||
|
||||
@@ -75,6 +67,13 @@ public class Document implements GenericSemanticNode {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Integer> getTreeId() {
|
||||
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void setTreeId(List<Integer> tocId) {
|
||||
|
||||
+8
-25
@@ -1,15 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -24,24 +21,20 @@ import lombok.experimental.FieldDefaults;
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Footer implements GenericSemanticNode {
|
||||
|
||||
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
|
||||
TextBlock leafTextBlock;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@@ -78,14 +71,4 @@ public class Footer implements GenericSemanticNode {
|
||||
return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = GenericSemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
public interface GenericSemanticNode extends SemanticNode {
|
||||
|
||||
+8
-25
@@ -1,15 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -24,24 +21,20 @@ import lombok.experimental.FieldDefaults;
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Header implements GenericSemanticNode {
|
||||
|
||||
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
|
||||
TextBlock leafTextBlock;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@@ -78,14 +71,4 @@ public class Header implements GenericSemanticNode {
|
||||
return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = GenericSemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
|
||||
}
|
||||
+8
-25
@@ -1,16 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -25,22 +22,18 @@ import lombok.experimental.FieldDefaults;
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Headline implements GenericSemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
TextBlock leafTextBlock;
|
||||
SectionIdentifier sectionIdentifier;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@@ -78,16 +71,6 @@ public class Headline implements GenericSemanticNode {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = GenericSemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public SectionIdentifier getSectionIdentifier() {
|
||||
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRuleHolder;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class Image implements GenericSemanticNode, MatchedRuleHolder {
|
||||
|
||||
List<Integer> treeId;
|
||||
String id;
|
||||
|
||||
ImageType imageType;
|
||||
boolean transparent;
|
||||
Rectangle2D position;
|
||||
|
||||
boolean removed;
|
||||
boolean ignored;
|
||||
|
||||
@Builder.Default
|
||||
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
Page page;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
public NodeType getType() {
|
||||
|
||||
return NodeType.IMAGE;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextBlock getTextBlock() {
|
||||
|
||||
return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Set<Page> getPages() {
|
||||
|
||||
return Collections.singleton(page);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
|
||||
bBoxPerPage.put(page, position);
|
||||
return bBoxPerPage;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-14
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
@@ -6,22 +6,10 @@ public enum ImageType {
|
||||
LOGO,
|
||||
FORMULA,
|
||||
SIGNATURE,
|
||||
OTHER {
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return "image";
|
||||
}
|
||||
},
|
||||
OTHER,
|
||||
OCR;
|
||||
|
||||
|
||||
public String toString() {
|
||||
|
||||
return name().toLowerCase(Locale.ENGLISH);
|
||||
}
|
||||
|
||||
|
||||
public static ImageType fromString(String imageType) {
|
||||
|
||||
return switch (imageType.toLowerCase(Locale.ROOT)) {
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
@@ -16,6 +16,6 @@ public enum NodeType {
|
||||
|
||||
public String toString() {
|
||||
|
||||
return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ENGLISH);
|
||||
return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
|
||||
}
|
||||
}
|
||||
+26
-7
@@ -1,12 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -23,23 +24,26 @@ import lombok.experimental.FieldDefaults;
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Page {
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
Integer number;
|
||||
Integer height;
|
||||
Integer width;
|
||||
Integer rotation;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
List<SemanticNode> mainBody;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Header header;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Footer footer;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<Image> images = new HashSet<>();
|
||||
|
||||
|
||||
@@ -54,4 +58,19 @@ public class Page {
|
||||
|
||||
return String.valueOf(number);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
|
||||
return number;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
|
||||
return o instanceof Page && o.hashCode() == this.hashCode();
|
||||
}
|
||||
|
||||
}
|
||||
+7
-25
@@ -1,15 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -22,22 +19,17 @@ import lombok.experimental.FieldDefaults;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Paragraph implements GenericSemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
TextBlock leafTextBlock;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@@ -67,14 +59,4 @@ public class Paragraph implements GenericSemanticNode {
|
||||
return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = GenericSemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
|
||||
}
|
||||
+8
-27
@@ -1,23 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@@ -26,22 +22,17 @@ import lombok.extern.slf4j.Slf4j;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Section implements GenericSemanticNode {
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
|
||||
TextBlock textBlock;
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
@@ -101,14 +92,4 @@ public class Section implements GenericSemanticNode {
|
||||
return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = GenericSemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
+58
-223
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@@ -9,20 +9,17 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility;
|
||||
|
||||
public interface SemanticNode {
|
||||
|
||||
@@ -46,11 +43,11 @@ public interface SemanticNode {
|
||||
|
||||
/**
|
||||
* Any Node maintains its own Set of Entities.
|
||||
* This Set contains all Entities whose TextRange intersects the TextRange of this node.
|
||||
* This Set contains all Entities whose boundary intersects the boundary of this node.
|
||||
*
|
||||
* @return Set of all Entities associated with this Node
|
||||
*/
|
||||
Set<TextEntity> getEntities();
|
||||
Set<RedactionEntity> getEntities();
|
||||
|
||||
|
||||
/**
|
||||
@@ -63,7 +60,6 @@ public interface SemanticNode {
|
||||
return getTextBlock().getPages();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Finds the first page associated with this Node.
|
||||
*
|
||||
@@ -76,26 +72,20 @@ public interface SemanticNode {
|
||||
|
||||
|
||||
/**
|
||||
* Each AtomicTextBlock is assigned a page, so to get the pages for this TextRange, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
|
||||
* Each AtomicTextBlock is assigned a page, so to get the pages for this boundary, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
|
||||
*
|
||||
* @return Set of PageNodes this node appears on.
|
||||
*/
|
||||
default Set<Page> getPages(TextRange textRange) {
|
||||
default Set<Page> getPages(Boundary boundary) {
|
||||
|
||||
if (!getTextRange().intersects(textRange)) {
|
||||
throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
|
||||
if (!getBoundary().contains(boundary)) {
|
||||
throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", boundary, getBoundary()));
|
||||
}
|
||||
return getTextBlock().getPages(textRange);
|
||||
return getTextBlock().getPages(boundary);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if the given page number exists in the list of pages.
|
||||
*
|
||||
* @param pageNumber the page number to be checked
|
||||
* @return true if the page number exists, otherwise false
|
||||
*/
|
||||
default boolean onPage(int pageNumber) {
|
||||
default boolean isOnPage(int pageNumber) {
|
||||
|
||||
return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
|
||||
}
|
||||
@@ -180,30 +170,6 @@ public interface SemanticNode {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the next sibling node of this SemanticNode in the document tree, if any.
|
||||
* If there is no next sibling node, an empty Optional is returned.
|
||||
*
|
||||
* @return Optional containing the next sibling node, or empty if there is none
|
||||
*/
|
||||
default Optional<SemanticNode> getNextSibling() {
|
||||
|
||||
return getDocumentTree().getNextSibling(getTreeId());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the previous sibling node of this SemanticNode in the document tree, if any.
|
||||
* If there is no previous sibling node, an empty Optional is returned.
|
||||
*
|
||||
* @return Optional containing the previous sibling node, or empty if there is none
|
||||
*/
|
||||
default Optional<SemanticNode> getPreviousSibling() {
|
||||
|
||||
return getDocumentTree().getPreviousSibling(getTreeId());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Leaf means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
|
||||
* Currently only Sections, Images, and Tables are not leaves.
|
||||
@@ -249,7 +215,7 @@ public interface SemanticNode {
|
||||
*/
|
||||
default boolean hasEntitiesOfType(String type) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
|
||||
return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@@ -258,11 +224,11 @@ public interface SemanticNode {
|
||||
* Ignores Entity with ignored == true or removed == true.
|
||||
*
|
||||
* @param types an array of strings representing the types of entities to check for
|
||||
* @return true, if this SemanticNode has at least one Entity of the provided types
|
||||
* @return true, if this SemanticNode has at least one Entity of any of the provided types
|
||||
*/
|
||||
default boolean hasEntitiesOfAnyType(String... types) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
|
||||
return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
|
||||
}
|
||||
|
||||
|
||||
@@ -275,7 +241,11 @@ public interface SemanticNode {
|
||||
*/
|
||||
default boolean hasEntitiesOfAllTypes(String... types) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
|
||||
return getEntities().stream()
|
||||
.filter(RedactionEntity::isActive)
|
||||
.map(RedactionEntity::getType)
|
||||
.collect(Collectors.toUnmodifiableSet())
|
||||
.containsAll(Arrays.stream(types).toList());
|
||||
}
|
||||
|
||||
|
||||
@@ -286,9 +256,9 @@ public interface SemanticNode {
|
||||
* @param type string representing the type of entities to return
|
||||
* @return List of RedactionEntities of any the type
|
||||
*/
|
||||
default List<TextEntity> getEntitiesOfType(String type) {
|
||||
default List<RedactionEntity> getEntitiesOfType(String type) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
|
||||
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
|
||||
}
|
||||
|
||||
|
||||
@@ -299,9 +269,9 @@ public interface SemanticNode {
|
||||
* @param types A list of strings representing the types of entities to return
|
||||
* @return List of RedactionEntities of any provided type
|
||||
*/
|
||||
default List<TextEntity> getEntitiesOfType(List<String> types) {
|
||||
default List<RedactionEntity> getEntitiesOfType(List<String> types) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
|
||||
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
|
||||
}
|
||||
|
||||
|
||||
@@ -312,9 +282,9 @@ public interface SemanticNode {
|
||||
* @param types A list of strings representing the types of entities to return
|
||||
* @return List of RedactionEntities that match any of the provided types
|
||||
*/
|
||||
default List<TextEntity> getEntitiesOfType(String... types) {
|
||||
default List<RedactionEntity> getEntitiesOfType(String... types) {
|
||||
|
||||
return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
|
||||
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
|
||||
}
|
||||
|
||||
|
||||
@@ -357,13 +327,6 @@ public interface SemanticNode {
|
||||
return getTextBlock().getSearchText().contains(string);
|
||||
}
|
||||
|
||||
Set<LayoutEngine> getEngines();
|
||||
|
||||
default void addEngine(LayoutEngine engine) {
|
||||
getEngines().add(engine);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains all the provided Strings.
|
||||
@@ -437,78 +400,6 @@ public interface SemanticNode {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains exactly the provided String as a word.
|
||||
*
|
||||
* @param word - String which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains string
|
||||
*/
|
||||
default boolean containsWord(String word) {
|
||||
|
||||
return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
|
||||
*
|
||||
* @param word - String which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains string
|
||||
*/
|
||||
default boolean containsWordIgnoreCase(String word) {
|
||||
|
||||
return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains any of the provided Strings as a word.
|
||||
*
|
||||
* @param words - A List of Strings which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains any of the provided strings
|
||||
*/
|
||||
default boolean containsAnyWord(String... words) {
|
||||
|
||||
return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
|
||||
*
|
||||
* @param words - A List of Strings which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains any of the provided strings
|
||||
*/
|
||||
default boolean containsAnyWordIgnoreCase(String... words) {
|
||||
|
||||
return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains all the provided Strings as word.
|
||||
*
|
||||
* @param words - A List of Strings which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains all the provided strings
|
||||
*/
|
||||
default boolean containsAllWords(String... words) {
|
||||
|
||||
return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
|
||||
*
|
||||
* @param words - A List of Strings which the TextBlock might contain
|
||||
* @return true, if this node's TextBlock contains all the provided strings
|
||||
*/
|
||||
default boolean containsAllWordsIgnoreCase(String... words) {
|
||||
|
||||
return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode matches the provided regex pattern.
|
||||
*
|
||||
@@ -534,37 +425,22 @@ public interface SemanticNode {
|
||||
|
||||
|
||||
/**
|
||||
* Checks whether this SemanticNode intersects the provided rectangle.
|
||||
*
|
||||
* @param x the lower left corner X value
|
||||
* @param y the lower left corner Y value
|
||||
* @param w width
|
||||
* @param h height
|
||||
* @param pageNumber the pageNumber of the rectangle
|
||||
* @return true if intersects, false otherwise
|
||||
*/
|
||||
default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
|
||||
|
||||
return getBBox().entrySet().stream().filter(entry -> entry.getKey().getNumber() == pageNumber).map(Map.Entry::getValue).anyMatch(rect -> rect.intersects(x, y, w, h));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This function is used during insertion of EntityNodes into the graph, it checks if the TextRange of the RedactionEntity intersects or even contains the RedactionEntity.
|
||||
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the RedactionEntity intersects or even contains the RedactionEntity.
|
||||
* It sets the fields accordingly and recursively calls this function on all its children.
|
||||
*
|
||||
* @param textEntity RedactionEntity, which is being inserted into the graph
|
||||
* @param redactionEntity RedactionEntity, which is being inserted into the graph
|
||||
*/
|
||||
default void addThisToEntityIfIntersects(TextEntity textEntity) {
|
||||
default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) {
|
||||
|
||||
TextBlock textBlock = getTextBlock();
|
||||
if (textBlock.getTextRange().intersects(textEntity.getTextRange())) {
|
||||
if (textBlock.containsTextRange(textEntity.getTextRange())) {
|
||||
textEntity.setDeepestFullyContainingNode(this);
|
||||
if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
|
||||
if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
|
||||
redactionEntity.setDeepestFullyContainingNode(this);
|
||||
}
|
||||
textEntity.addIntersectingNode(this);
|
||||
streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
|
||||
.forEach(node -> node.addThisToEntityIfIntersects(textEntity));
|
||||
|
||||
redactionEntity.addIntersectingNode(this);
|
||||
streamChildren().filter(semanticNode -> semanticNode.getBoundary().intersects(redactionEntity.getBoundary()))
|
||||
.forEach(node -> node.addThisToEntityIfIntersects(redactionEntity));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -614,13 +490,13 @@ public interface SemanticNode {
|
||||
|
||||
|
||||
/**
|
||||
* The TextRange is the start and end string offsets in the reading order of the document.
|
||||
* The Boundary is the start and end string offsets in the reading order of the document.
|
||||
*
|
||||
* @return TextRange of this Node's TextBlock
|
||||
* @return Boundary of this Node's TextBlock
|
||||
*/
|
||||
default TextRange getTextRange() {
|
||||
default Boundary getBoundary() {
|
||||
|
||||
return getTextBlock().getTextRange();
|
||||
return getTextBlock().getBoundary();
|
||||
}
|
||||
|
||||
|
||||
@@ -631,28 +507,7 @@ public interface SemanticNode {
|
||||
*/
|
||||
default int length() {
|
||||
|
||||
return getTextRange().length();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* For a given TextRange this function returns a List of rectangle around the text in the range.
|
||||
* These Rectangles are split either by a new line or by a large gap in the current line.
|
||||
* This is mainly used to find the positions of TextEntities
|
||||
*
|
||||
* @param textRange A TextRange to calculate the positions for.
|
||||
* @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
|
||||
*/
|
||||
default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
|
||||
|
||||
if (isLeaf()) {
|
||||
return getTextBlock().getPositionsPerPage(textRange);
|
||||
}
|
||||
Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
|
||||
if (containingChildNode.isEmpty()) {
|
||||
return getTextBlock().getPositionsPerPage(textRange);
|
||||
}
|
||||
return containingChildNode.get().getPositionsPerPage(textRange);
|
||||
return getBoundary().length();
|
||||
}
|
||||
|
||||
|
||||
@@ -664,11 +519,12 @@ public interface SemanticNode {
|
||||
*/
|
||||
default Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
|
||||
if (isLeaf()) {
|
||||
return getBBoxFromLeafTextBlock();
|
||||
return getBBoxFromLeafTextBlock(bBoxPerPage);
|
||||
}
|
||||
|
||||
return getBBoxFromChildren();
|
||||
return getBBoxFromChildren(bBoxPerPage);
|
||||
}
|
||||
|
||||
|
||||
@@ -691,49 +547,28 @@ public interface SemanticNode {
|
||||
|
||||
/**
|
||||
* TODO: this produces unwanted results for sections spanning multiple columns.
|
||||
* Computes the Union of the bounding boxes of all children recursively.
|
||||
*
|
||||
* @param bBoxPerPage initial empty BoundingBox
|
||||
* @return The union of the BoundingBoxes of all children
|
||||
*/
|
||||
private Map<Page, Rectangle2D> getBBoxFromChildren() {
|
||||
private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
|
||||
|
||||
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
|
||||
List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
|
||||
Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
|
||||
for (Page page : pages) {
|
||||
Rectangle2D bBoxOnPage = childrenBBoxes.stream()
|
||||
.filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
|
||||
.map(childBboxPerPage -> childBboxPerPage.get(page))
|
||||
.collect(RectangleTransformations.collectBBox());
|
||||
bBoxPerPage.put(page, bBoxOnPage);
|
||||
}
|
||||
return bBoxPerPage;
|
||||
return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
|
||||
map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
|
||||
return map2;
|
||||
}).orElse(bBoxPerPage);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param bBoxPerPage initial empty BoundingBox
|
||||
* @return The union of all BoundingBoxes of the TextBlock of this node
|
||||
*/
|
||||
private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
|
||||
private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
|
||||
|
||||
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
|
||||
Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
|
||||
atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
|
||||
atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs)));
|
||||
return bBoxPerPage;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
|
||||
* The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
|
||||
*
|
||||
* @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
|
||||
* @see NodeVisitor
|
||||
*/
|
||||
default void accept(NodeVisitor visitor) {
|
||||
|
||||
visitor.visit(this);
|
||||
streamChildren().forEach(childNode -> childNode.accept(visitor));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+44
-149
@@ -1,23 +1,19 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -30,12 +26,8 @@ import lombok.experimental.FieldDefaults;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Table implements SemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
DocumentTree documentTree;
|
||||
|
||||
@@ -45,19 +37,8 @@ public class Table implements SemanticNode {
|
||||
TextBlock textBlock;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
Map<Page, Rectangle2D> bBoxCache;
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
if (bBoxCache == null) {
|
||||
bBoxCache = SemanticNode.super.getBBox();
|
||||
}
|
||||
return bBoxCache;
|
||||
}
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
/**
|
||||
@@ -66,9 +47,10 @@ public class Table implements SemanticNode {
|
||||
* @param strings Strings to check whether a row contains them
|
||||
* @return Stream of all entities in this table, that appear in a row, which contains any of the provided strings
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
|
||||
public Stream<RedactionEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
|
||||
|
||||
return IntStream.range(0, numberOfRows).boxed()
|
||||
return IntStream.range(0, numberOfRows)
|
||||
.boxed()
|
||||
.filter(row -> rowContainsStringsIgnoreCase(row, strings))
|
||||
.flatMap(this::streamRow)
|
||||
.map(TableCell::getEntities)
|
||||
@@ -85,11 +67,8 @@ public class Table implements SemanticNode {
|
||||
*/
|
||||
public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
|
||||
|
||||
String rowText = streamRow(row).map(TableCell::getTextBlock)
|
||||
.collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
|
||||
return strings.stream()
|
||||
.map(String::toLowerCase)
|
||||
.allMatch(rowText::contains);
|
||||
String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
|
||||
return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
|
||||
}
|
||||
|
||||
|
||||
@@ -100,15 +79,11 @@ public class Table implements SemanticNode {
|
||||
* @param value the string which the table cell should contain
|
||||
* @return a stream of all entities, which appear in a row where at least one cell has the provided header and the provided value.
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
|
||||
public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
|
||||
|
||||
List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
|
||||
.map(TableCell::getCol)
|
||||
.toList();
|
||||
List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
|
||||
return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
|
||||
.anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
|
||||
.map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream);
|
||||
.anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
@@ -119,15 +94,11 @@ public class Table implements SemanticNode {
|
||||
* @param values the strings which the table cell should contain
|
||||
* @return a stream of all entities, which appear in a row where at least one cell has the provided header and any provided value.
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
|
||||
public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
|
||||
|
||||
List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
|
||||
.map(TableCell::getCol)
|
||||
.toList();
|
||||
List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
|
||||
return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
|
||||
.anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
|
||||
.map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream);
|
||||
.anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
@@ -138,35 +109,18 @@ public class Table implements SemanticNode {
|
||||
* @param types type strings to check whether a row contains an entity like them
|
||||
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
|
||||
public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
|
||||
|
||||
return IntStream.range(0, numberOfRows).boxed()
|
||||
.filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
|
||||
.anyMatch(types::contains))
|
||||
.flatMap(this::streamRow)
|
||||
.map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream);
|
||||
}
|
||||
List<Integer> rowsWithEntityOfType = getEntities().stream()
|
||||
.filter(RedactionEntity::isActive)
|
||||
.filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
|
||||
.map(RedactionEntity::getIntersectingNodes)
|
||||
.filter(node -> node instanceof TableCell)
|
||||
.map(node -> (TableCell) node)
|
||||
.map(TableCell::getRow)
|
||||
.toList();
|
||||
|
||||
|
||||
/**
|
||||
* Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
|
||||
* Ignores Entity with ignored == true or removed == true.
|
||||
*
|
||||
* @param types type strings to check whether a row contains an entity like them
|
||||
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
|
||||
|
||||
return IntStream.range(0, numberOfRows).boxed()
|
||||
.filter(rowNumber -> {
|
||||
Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
|
||||
.collect(Collectors.toSet());
|
||||
return entityTypes.containsAll(types);
|
||||
})
|
||||
.flatMap(this::streamRow)
|
||||
.map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream);
|
||||
return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
@@ -177,45 +131,20 @@ public class Table implements SemanticNode {
|
||||
* @param types type strings to check whether a row contains an entity like them
|
||||
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
|
||||
*/
|
||||
public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
|
||||
public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
|
||||
|
||||
return IntStream.range(0, numberOfRows).boxed()
|
||||
.filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
|
||||
.noneMatch(types::contains))
|
||||
return IntStream.range(0, numberOfRows)
|
||||
.boxed()
|
||||
.filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream)
|
||||
.filter(RedactionEntity::isActive)
|
||||
.noneMatch(entity -> types.contains(entity.getType())))
|
||||
.flatMap(this::streamRow)
|
||||
.map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Streams all Entities in the given row.
|
||||
*
|
||||
* @param rowNumber the row number to look for
|
||||
* @return stream of TextEntities occurring in row
|
||||
*/
|
||||
public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
|
||||
|
||||
return streamRow(rowNumber).map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream)
|
||||
.filter(TextEntity::active);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Streams all Entities in the given col.
|
||||
*
|
||||
* @param colNumber the column number to look for
|
||||
* @return stream of TextEntities occurring in row
|
||||
*/
|
||||
public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
|
||||
|
||||
return streamCol(colNumber).map(TableCell::getEntities)
|
||||
.flatMap(Collection::stream)
|
||||
.filter(TextEntity::active);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns a TableCell at the provided row and column location.
|
||||
*
|
||||
@@ -229,8 +158,7 @@ public class Table implements SemanticNode {
|
||||
throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
|
||||
}
|
||||
int idx = row * numberOfCols + col;
|
||||
return (TableCell) documentTree.getEntryById(treeId).getChildren()
|
||||
.get(idx).getNode();
|
||||
return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
|
||||
}
|
||||
|
||||
|
||||
@@ -245,18 +173,6 @@ public class Table implements SemanticNode {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Streams all TableCells that contain at least one entity of a given type in this Table row-wise.
|
||||
*
|
||||
* @param type The type of the entity
|
||||
* @return Stream of filtered TableCells
|
||||
*/
|
||||
public Stream<TableCell> streamTableCellsWhichContainType(String type) {
|
||||
|
||||
return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Streams all TableCells in this Table which have the provided header row-wise.
|
||||
*
|
||||
@@ -279,8 +195,7 @@ public class Table implements SemanticNode {
|
||||
*/
|
||||
public Stream<TableCell> streamCol(int col) {
|
||||
|
||||
return IntStream.range(0, numberOfRows).boxed()
|
||||
.map(row -> getCell(row, col));
|
||||
return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
|
||||
}
|
||||
|
||||
|
||||
@@ -292,8 +207,7 @@ public class Table implements SemanticNode {
|
||||
*/
|
||||
public Stream<TableCell> streamRow(int row) {
|
||||
|
||||
return IntStream.range(0, numberOfCols).boxed()
|
||||
.map(col -> getCell(row, col));
|
||||
return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
|
||||
}
|
||||
|
||||
|
||||
@@ -317,8 +231,7 @@ public class Table implements SemanticNode {
|
||||
*/
|
||||
public Stream<TableCell> streamHeadersForCell(int row, int col) {
|
||||
|
||||
return Stream.concat(streamRow(row), streamCol(col))
|
||||
.filter(TableCell::isHeader);
|
||||
return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
|
||||
}
|
||||
|
||||
|
||||
@@ -334,22 +247,6 @@ public class Table implements SemanticNode {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Streams all Headers and checks if any equal the provided string.
|
||||
*
|
||||
* @param header string to check the headers for
|
||||
* @return true, if at least one header equals the provided string
|
||||
*/
|
||||
public boolean hasHeaderIgnoreCase(String header) {
|
||||
|
||||
return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock()
|
||||
.getSearchText()
|
||||
.strip()
|
||||
.toLowerCase(Locale.ENGLISH)
|
||||
.equals(header.toLowerCase(Locale.ENGLISH)));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if this table has a column with the provided header and any of the table cells in that column contain the provided value.
|
||||
*
|
||||
@@ -380,13 +277,13 @@ public class Table implements SemanticNode {
|
||||
* Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
|
||||
* Ignores Entity with ignored == true or removed == true.
|
||||
*
|
||||
* @param type the type of entities to search for
|
||||
* @param textEntity the entity, which appears in the row to search
|
||||
* @param type the type of entities to search for
|
||||
* @param redactionEntity the entity, which appears in the row to search
|
||||
* @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
|
||||
*/
|
||||
public List<TextEntity> getEntitiesOfTypeInSameRow(String type, TextEntity textEntity) {
|
||||
public List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
|
||||
|
||||
return textEntity.getIntersectingNodes()
|
||||
return redactionEntity.getIntersectingNodes()
|
||||
.stream()
|
||||
.filter(node -> node instanceof TableCell)
|
||||
.map(node -> (TableCell) node)
|
||||
@@ -408,9 +305,7 @@ public class Table implements SemanticNode {
|
||||
public TextBlock getTextBlock() {
|
||||
|
||||
if (textBlock == null) {
|
||||
textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
|
||||
.map(SemanticNode::getLeafTextBlock)
|
||||
.collect(new TextBlockCollector());
|
||||
textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
|
||||
}
|
||||
return textBlock;
|
||||
}
|
||||
+8
-12
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.HashMap;
|
||||
@@ -7,11 +7,10 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -24,13 +23,8 @@ import lombok.experimental.FieldDefaults;
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class TableCell implements GenericSemanticNode {
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
int row;
|
||||
int col;
|
||||
@@ -42,10 +36,12 @@ public class TableCell implements GenericSemanticNode {
|
||||
|
||||
TextBlock textBlock;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<RedactionEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
+36
-54
@@ -1,24 +1,21 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.text.BreakIterator;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
|
||||
|
||||
@@ -41,9 +38,8 @@ public class AtomicTextBlock implements TextBlock {
|
||||
Page page;
|
||||
|
||||
//string coordinates
|
||||
TextRange textRange;
|
||||
Boundary boundary;
|
||||
String searchText;
|
||||
List<String> words;
|
||||
List<Integer> lineBreaks;
|
||||
|
||||
//position coordinates
|
||||
@@ -65,7 +61,7 @@ public class AtomicTextBlock implements TextBlock {
|
||||
|
||||
return AtomicTextBlock.builder()
|
||||
.id(textBlockIdx)
|
||||
.textRange(new TextRange(stringOffset, stringOffset))
|
||||
.boundary(new Boundary(stringOffset, stringOffset))
|
||||
.searchText("")
|
||||
.lineBreaks(Collections.emptyList())
|
||||
.page(page)
|
||||
@@ -86,7 +82,7 @@ public class AtomicTextBlock implements TextBlock {
|
||||
.id(atomicTextBlockData.getId())
|
||||
.numberOnPage(atomicTextBlockData.getNumberOnPage())
|
||||
.page(page)
|
||||
.textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
|
||||
.boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
|
||||
.searchText(atomicTextBlockData.getSearchText())
|
||||
.lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
|
||||
.stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
|
||||
@@ -102,34 +98,20 @@ public class AtomicTextBlock implements TextBlock {
|
||||
}
|
||||
|
||||
|
||||
public TextRange getLineTextRange(int lineNumber) {
|
||||
public Boundary getLineBoundary(int lineNumber) {
|
||||
|
||||
if (lineNumber >= numberOfLines() || lineNumber < 0) {
|
||||
return new TextRange(textRange.start(), textRange.start());
|
||||
return new Boundary(boundary.start(), boundary.start());
|
||||
}
|
||||
if (numberOfLines() == 1) {
|
||||
return textRange;
|
||||
return boundary;
|
||||
}
|
||||
if (lineNumber == 0) {
|
||||
return new TextRange(textRange.start(), lineBreaks.get(0) + textRange.start());
|
||||
return new Boundary(boundary.start(), lineBreaks.get(0) + boundary.start());
|
||||
} else if (lineNumber == numberOfLines() - 1) {
|
||||
return new TextRange(lineBreaks.get(lineBreaks.size() - 1) + textRange.start(), textRange.end());
|
||||
return new Boundary(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end());
|
||||
}
|
||||
return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
|
||||
}
|
||||
|
||||
public List<String> getWords() {
|
||||
|
||||
if (words == null) {
|
||||
words = new ArrayList<>();
|
||||
BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
|
||||
iterator.setText(searchText);
|
||||
int start = iterator.first();
|
||||
for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
|
||||
words.add(searchText.substring(start, end));
|
||||
}
|
||||
}
|
||||
return words;
|
||||
return new Boundary(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start());
|
||||
}
|
||||
|
||||
|
||||
@@ -144,9 +126,9 @@ public class AtomicTextBlock implements TextBlock {
|
||||
public int getNextLinebreak(int fromIndex) {
|
||||
|
||||
return lineBreaks.stream()//
|
||||
.filter(linebreak -> linebreak > fromIndex - textRange.start()) //
|
||||
.filter(linebreak -> linebreak > fromIndex - boundary.start()) //
|
||||
.findFirst() //
|
||||
.orElse(searchText.length()) + textRange.start();
|
||||
.orElse(searchText.length()) + boundary.start();
|
||||
}
|
||||
|
||||
|
||||
@@ -154,43 +136,43 @@ public class AtomicTextBlock implements TextBlock {
|
||||
public int getPreviousLinebreak(int fromIndex) {
|
||||
|
||||
return lineBreaks.stream()//
|
||||
.filter(linebreak -> linebreak <= fromIndex - textRange.start())//
|
||||
.filter(linebreak -> linebreak <= fromIndex - boundary.start())//
|
||||
.reduce((a, b) -> b)//
|
||||
.orElse(0) + textRange.start();
|
||||
.orElse(0) + boundary.start();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Rectangle2D getPosition(int stringIdx) {
|
||||
|
||||
return positions.get(stringIdxToPositionIdx.get(stringIdx - textRange.start()));
|
||||
return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start()));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Rectangle2D> getPositions(TextRange stringTextRange) {
|
||||
public List<Rectangle2D> getPositions(Boundary stringBoundary) {
|
||||
|
||||
if (!containsTextRange(stringTextRange)) {
|
||||
throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringTextRange, this.textRange));
|
||||
if (!containsBoundary(stringBoundary)) {
|
||||
throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringBoundary, this.boundary));
|
||||
}
|
||||
if (stringTextRange.length() == 0) {
|
||||
if (stringBoundary.length() == 0) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
int startPositionIdx = stringIdxToPositionIdx.get(stringTextRange.start() - this.textRange.start());
|
||||
int startPositionIdx = stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start());
|
||||
|
||||
if (stringTextRange.end() == this.textRange.end()) {
|
||||
if (stringBoundary.end() == this.boundary.end()) {
|
||||
return positions.subList(startPositionIdx, positions.size());
|
||||
}
|
||||
|
||||
return positions.subList(startPositionIdx, stringIdxToPositionIdx.get(stringTextRange.end() - this.textRange.start()));
|
||||
return positions.subList(startPositionIdx, stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start()));
|
||||
|
||||
}
|
||||
|
||||
|
||||
public Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange) {
|
||||
public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
|
||||
|
||||
List<Rectangle2D> rectanglesPerLine = stringTextRange.split(getAllLineBreaksInBoundary(stringTextRange))
|
||||
List<Rectangle2D> rectanglesPerLine = stringBoundary.split(getAllLineBreaksInBoundary(stringBoundary))
|
||||
.stream()
|
||||
.map(this::getPositions)
|
||||
.map(RectangleTransformations::rectangleBBoxWithGaps)
|
||||
@@ -203,18 +185,18 @@ public class AtomicTextBlock implements TextBlock {
|
||||
|
||||
|
||||
@Override
|
||||
public String subSequenceWithLineBreaks(TextRange textRange) {
|
||||
public String subSequenceWithLineBreaks(Boundary boundary) {
|
||||
|
||||
if (textRange.length() == 0 || !getTextRange().contains(textRange)) {
|
||||
if (boundary.length() == 0 || !getBoundary().contains(boundary)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + textRange.start()).filter(textRange::contains).collect(Collectors.toSet());
|
||||
if (textRange.end() == getTextRange().end()) {
|
||||
lbInBoundary.add(getTextRange().end());
|
||||
Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + boundary.start()).filter(boundary::contains).collect(Collectors.toSet());
|
||||
if (boundary.end() == getBoundary().end()) {
|
||||
lbInBoundary.add(getBoundary().end());
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = textRange.start(); i < textRange.end(); i++) {
|
||||
for (int i = boundary.start(); i < boundary.end(); i++) {
|
||||
char character = this.charAt(i);
|
||||
if (lbInBoundary.contains(i + 1)) {
|
||||
// always plus one, due to the linebreaks being an exclusive end index
|
||||
@@ -233,9 +215,9 @@ public class AtomicTextBlock implements TextBlock {
|
||||
}
|
||||
|
||||
|
||||
private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
|
||||
private List<Integer> getAllLineBreaksInBoundary(Boundary boundary) {
|
||||
|
||||
return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
|
||||
return getLineBreaks().stream().map(linebreak -> linebreak + this.boundary.start()).filter(boundary::contains).toList();
|
||||
}
|
||||
|
||||
|
||||
+37
-54
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@@ -7,12 +7,11 @@ import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Data;
|
||||
@@ -24,7 +23,7 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
|
||||
List<AtomicTextBlock> atomicTextBlocks;
|
||||
String searchText;
|
||||
TextRange textRange;
|
||||
Boundary boundary;
|
||||
|
||||
|
||||
public static ConcatenatedTextBlock empty() {
|
||||
@@ -37,12 +36,12 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
|
||||
this.atomicTextBlocks = new LinkedList<>();
|
||||
if (atomicTextBlocks.isEmpty()) {
|
||||
textRange = new TextRange(-1, -1);
|
||||
boundary = new Boundary(-1, -1);
|
||||
return;
|
||||
}
|
||||
var firstTextBlock = atomicTextBlocks.get(0);
|
||||
this.atomicTextBlocks.add(firstTextBlock);
|
||||
textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end());
|
||||
boundary = new Boundary(firstTextBlock.getBoundary().start(), firstTextBlock.getBoundary().end());
|
||||
|
||||
atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
|
||||
}
|
||||
@@ -51,13 +50,13 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
public ConcatenatedTextBlock concat(TextBlock textBlock) {
|
||||
|
||||
if (this.atomicTextBlocks.isEmpty()) {
|
||||
textRange.setStart(textBlock.getTextRange().start());
|
||||
textRange.setEnd(textBlock.getTextRange().end());
|
||||
} else if (textRange.end() != textBlock.getTextRange().start()) {
|
||||
throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", textRange, textBlock.getTextRange()));
|
||||
boundary.setStart(textBlock.getBoundary().start());
|
||||
boundary.setEnd(textBlock.getBoundary().end());
|
||||
} else if (boundary.end() != textBlock.getBoundary().start()) {
|
||||
throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", boundary, textBlock.getBoundary()));
|
||||
}
|
||||
this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
|
||||
textRange.setEnd(textBlock.getTextRange().end());
|
||||
boundary.setEnd(textBlock.getBoundary().end());
|
||||
this.searchText = null;
|
||||
return this;
|
||||
}
|
||||
@@ -65,22 +64,13 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
|
||||
private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
|
||||
|
||||
return atomicTextBlocks.stream().filter(textBlock -> textBlock.getTextRange().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
|
||||
return atomicTextBlocks.stream().filter(textBlock -> textBlock.getBoundary().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
|
||||
}
|
||||
|
||||
|
||||
private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
|
||||
private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(Boundary boundary) {
|
||||
|
||||
List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
|
||||
for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
|
||||
if (atomicTextBlock.getTextRange().start() > textRange.end()) {
|
||||
break; // early stop, following TextBlocks will never intersect
|
||||
}
|
||||
if (atomicTextBlock.getTextRange().intersects(textRange)) {
|
||||
intersectingAtomicTextBlocks.add(atomicTextBlock);
|
||||
}
|
||||
}
|
||||
return intersectingAtomicTextBlocks;
|
||||
return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList();
|
||||
}
|
||||
|
||||
|
||||
@@ -96,13 +86,6 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<String> getWords() {
|
||||
|
||||
return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int numberOfLines() {
|
||||
|
||||
@@ -131,106 +114,106 @@ public class ConcatenatedTextBlock implements TextBlock {
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public Rectangle2D getPosition(int stringIdx) {
|
||||
|
||||
return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
|
||||
}
|
||||
|
||||
|
||||
public TextRange getLineTextRange(int lineNumber) {
|
||||
public Boundary getLineBoundary(int lineNumber) {
|
||||
|
||||
if (atomicTextBlocks.size() == 1) {
|
||||
return atomicTextBlocks.get(0).getLineTextRange(lineNumber);
|
||||
return atomicTextBlocks.get(0).getLineBoundary(lineNumber);
|
||||
}
|
||||
int lineNumberInCurrentBlock = lineNumber;
|
||||
for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
|
||||
if (lineNumberInCurrentBlock < atomicTextBlock.numberOfLines()) {
|
||||
return atomicTextBlock.getLineTextRange(lineNumberInCurrentBlock);
|
||||
return atomicTextBlock.getLineBoundary(lineNumberInCurrentBlock);
|
||||
}
|
||||
lineNumberInCurrentBlock -= atomicTextBlock.numberOfLines();
|
||||
}
|
||||
return new TextRange(textRange.start(), textRange.start());
|
||||
return new Boundary(boundary.start(), boundary.start());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<Rectangle2D> getPositions(TextRange stringTextRange) {
|
||||
public List<Rectangle2D> getPositions(Boundary stringBoundary) {
|
||||
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
|
||||
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
|
||||
|
||||
if (textBlocks.isEmpty()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
if (textBlocks.size() == 1) {
|
||||
return textBlocks.get(0).getPositions(stringTextRange);
|
||||
return textBlocks.get(0).getPositions(stringBoundary);
|
||||
}
|
||||
|
||||
AtomicTextBlock firstTextBlock = textBlocks.get(0);
|
||||
List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new TextRange(stringTextRange.start(), firstTextBlock.getTextRange().end())));
|
||||
List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
|
||||
|
||||
for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
|
||||
positions.addAll(textBlock.getPositions());
|
||||
}
|
||||
|
||||
var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
|
||||
positions.addAll(lastTextBlock.getPositions(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end())));
|
||||
positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
|
||||
|
||||
return positions;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange) {
|
||||
public Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary) {
|
||||
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
|
||||
|
||||
if (textBlocks.isEmpty()) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
|
||||
if (textBlocks.size() == 1) {
|
||||
return textBlocks.get(0).getPositionsPerPage(stringTextRange);
|
||||
return textBlocks.get(0).getPositionsPerPage(stringBoundary);
|
||||
}
|
||||
|
||||
AtomicTextBlock firstTextBlock = textBlocks.get(0);
|
||||
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getPositionsPerPage(new TextRange(stringTextRange.start(), firstTextBlock.getTextRange().end()));
|
||||
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = firstTextBlock.getPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()));
|
||||
|
||||
for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
|
||||
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getPositionsPerPage(textBlock.getTextRange()));
|
||||
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, textBlock.getPositionsPerPage(textBlock.getBoundary()));
|
||||
}
|
||||
|
||||
AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
|
||||
rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage,
|
||||
lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end())));
|
||||
lastTextBlock.getPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
|
||||
|
||||
return rectanglesPerLinePerPage;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String subSequenceWithLineBreaks(TextRange textRange) {
|
||||
public String subSequenceWithLineBreaks(Boundary boundary) {
|
||||
|
||||
if (textRange.length() == 0 || !getTextRange().contains(textRange)) {
|
||||
if (boundary.length() == 0 || !getBoundary().contains(boundary)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(textRange);
|
||||
List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(boundary);
|
||||
|
||||
if (textBlocks.size() == 1) {
|
||||
return textBlocks.get(0).subSequenceWithLineBreaks(textRange);
|
||||
return textBlocks.get(0).subSequenceWithLineBreaks(boundary);
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
AtomicTextBlock firstTextBlock = textBlocks.get(0);
|
||||
sb.append(firstTextBlock.subSequenceWithLineBreaks(new TextRange(textRange.start(), firstTextBlock.getTextRange().end())));
|
||||
sb.append(firstTextBlock.subSequenceWithLineBreaks(new Boundary(boundary.start(), firstTextBlock.getBoundary().end())));
|
||||
|
||||
for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
|
||||
sb.append(textBlock.searchTextWithLineBreaks());
|
||||
}
|
||||
|
||||
var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
|
||||
sb.append(lastTextBlock.subSequenceWithLineBreaks(new TextRange(lastTextBlock.getTextRange().start(), textRange.end())));
|
||||
sb.append(lastTextBlock.subSequenceWithLineBreaks(new Boundary(lastTextBlock.getBoundary().start(), boundary.end())));
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
+28
-29
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
@@ -10,20 +10,19 @@ import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
|
||||
|
||||
public interface TextBlock extends CharSequence {
|
||||
|
||||
String getSearchText();
|
||||
|
||||
List<String> getWords();
|
||||
|
||||
List<AtomicTextBlock> getAtomicTextBlocks();
|
||||
|
||||
|
||||
TextRange getTextRange();
|
||||
Boundary getBoundary();
|
||||
|
||||
|
||||
int getNextLinebreak(int fromIndex);
|
||||
@@ -32,7 +31,7 @@ public interface TextBlock extends CharSequence {
|
||||
int getPreviousLinebreak(int fromIndex);
|
||||
|
||||
|
||||
TextRange getLineTextRange(int lineNumber);
|
||||
Boundary getLineBoundary(int lineNumber);
|
||||
|
||||
|
||||
|
||||
@@ -42,13 +41,13 @@ public interface TextBlock extends CharSequence {
|
||||
Rectangle2D getPosition(int stringIdx);
|
||||
|
||||
|
||||
List<Rectangle2D> getPositions(TextRange stringTextRange);
|
||||
List<Rectangle2D> getPositions(Boundary stringBoundary);
|
||||
|
||||
|
||||
Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange);
|
||||
Map<Page, List<Rectangle2D>> getPositionsPerPage(Boundary stringBoundary);
|
||||
|
||||
|
||||
String subSequenceWithLineBreaks(TextRange textRange);
|
||||
String subSequenceWithLineBreaks(Boundary boundary);
|
||||
|
||||
|
||||
int numberOfLines();
|
||||
@@ -56,13 +55,13 @@ public interface TextBlock extends CharSequence {
|
||||
|
||||
default CharSequence getLine(int lineNumber) {
|
||||
|
||||
return subSequence(getLineTextRange(lineNumber));
|
||||
return subSequence(getLineBoundary(lineNumber));
|
||||
}
|
||||
|
||||
|
||||
default List<Rectangle2D> getLinePositions(int lineNumber) {
|
||||
|
||||
return getPositions(getLineTextRange(lineNumber));
|
||||
return getPositions(getLineBoundary(lineNumber));
|
||||
}
|
||||
|
||||
|
||||
@@ -73,13 +72,13 @@ public interface TextBlock extends CharSequence {
|
||||
|
||||
default String searchTextWithLineBreaks() {
|
||||
|
||||
return subSequenceWithLineBreaks(getTextRange());
|
||||
return subSequenceWithLineBreaks(getBoundary());
|
||||
}
|
||||
|
||||
|
||||
default int indexOf(String searchTerm) {
|
||||
|
||||
return indexOf(searchTerm, getTextRange().start());
|
||||
return indexOf(searchTerm, getBoundary().start());
|
||||
}
|
||||
|
||||
|
||||
@@ -89,10 +88,10 @@ public interface TextBlock extends CharSequence {
|
||||
}
|
||||
|
||||
|
||||
default Set<Page> getPages(TextRange textRange) {
|
||||
default Set<Page> getPages(Boundary boundary) {
|
||||
|
||||
return getAtomicTextBlocks().stream()
|
||||
.filter(atomicTextBlock -> atomicTextBlock.getTextRange().intersects(textRange))
|
||||
.filter(atomicTextBlock -> atomicTextBlock.getBoundary().intersects(boundary))
|
||||
.map(AtomicTextBlock::getPage)
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
@@ -100,38 +99,38 @@ public interface TextBlock extends CharSequence {
|
||||
|
||||
default int indexOf(String searchTerm, int startOffset) {
|
||||
|
||||
int start = getSearchText().indexOf(searchTerm, startOffset - getTextRange().start());
|
||||
int start = getSearchText().indexOf(searchTerm, startOffset - getBoundary().start());
|
||||
if (start == -1) {
|
||||
return -1;
|
||||
}
|
||||
return start + getTextRange().start();
|
||||
return start + getBoundary().start();
|
||||
}
|
||||
|
||||
|
||||
default CharSequence getFirstLine() {
|
||||
|
||||
return subSequence(getTextRange().start(), getNextLinebreak(getTextRange().start()));
|
||||
return subSequence(getBoundary().start(), getNextLinebreak(getBoundary().start()));
|
||||
}
|
||||
|
||||
|
||||
default boolean containsTextRange(TextRange textRange) {
|
||||
default boolean containsBoundary(Boundary boundary) {
|
||||
|
||||
if (textRange.end() < textRange.start()) {
|
||||
throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", textRange));
|
||||
if (boundary.end() < boundary.start()) {
|
||||
throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", boundary));
|
||||
}
|
||||
return getTextRange().contains(textRange);
|
||||
return getBoundary().contains(boundary);
|
||||
}
|
||||
|
||||
|
||||
default boolean containsIndex(int stringIndex) {
|
||||
|
||||
return getTextRange().contains(stringIndex);
|
||||
return getBoundary().contains(stringIndex);
|
||||
}
|
||||
|
||||
|
||||
default CharSequence subSequence(TextRange textRange) {
|
||||
default CharSequence subSequence(Boundary boundary) {
|
||||
|
||||
return subSequence(textRange.start(), textRange.end());
|
||||
return subSequence(boundary.start(), boundary.end());
|
||||
}
|
||||
|
||||
|
||||
@@ -148,21 +147,21 @@ public interface TextBlock extends CharSequence {
|
||||
@Override
|
||||
default CharSequence subSequence(int start, int end) {
|
||||
|
||||
return getSearchText().substring(start - getTextRange().start(), end - getTextRange().start());
|
||||
return getSearchText().substring(start - getBoundary().start(), end - getBoundary().start());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
default int length() {
|
||||
|
||||
return getTextRange().length();
|
||||
return getBoundary().length();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
default char charAt(int index) {
|
||||
|
||||
return getSearchText().charAt(index - getTextRange().start());
|
||||
return getSearchText().charAt(index - getBoundary().start());
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.textblock;
|
||||
package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.function.BiConsumer;
|
||||
+664
@@ -0,0 +1,664 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.kie.api.runtime.KieSession;
|
||||
|
||||
import com.google.common.base.Functions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCell;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RedactionSearchUtility;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.ConsecutiveBoundaryCollector;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class EntityCreationService {
|
||||
|
||||
private final EntityEnrichmentService entityEnrichmentService;
|
||||
private final KieSession kieSession;
|
||||
|
||||
|
||||
public EntityCreationService(EntityEnrichmentService entityEnrichmentService) {
|
||||
|
||||
this.entityEnrichmentService = entityEnrichmentService;
|
||||
this.kieSession = null;
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
|
||||
|
||||
startBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() - start.length());
|
||||
boundary.setEnd(boundary.end() - start.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
|
||||
|
||||
startBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() - start.length());
|
||||
boundary.setEnd(boundary.end() - start.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
|
||||
|
||||
stopBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() + stop.length());
|
||||
boundary.setEnd(boundary.end() + stop.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
|
||||
|
||||
stopBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() + stop.length());
|
||||
boundary.setEnd(boundary.end() + stop.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
|
||||
|
||||
startBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() - start.length());
|
||||
boundary.setEnd(boundary.end() - start.length());
|
||||
});
|
||||
stopBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() + stop.length());
|
||||
boundary.setEnd(boundary.end() + stop.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
|
||||
|
||||
startBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() - start.length());
|
||||
boundary.setEnd(boundary.end() - start.length());
|
||||
});
|
||||
stopBoundaries.forEach(boundary -> {
|
||||
boundary.setStart(boundary.start() + stop.length());
|
||||
boundary.setEnd(boundary.end() + stop.length());
|
||||
});
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.getTextBlock();
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStart, textBlock);
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStop, textBlock);
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.getTextBlock();
|
||||
List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStart, 0, textBlock);
|
||||
List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStop, 0, textBlock);
|
||||
|
||||
return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> betweenBoundaries(List<Boundary> startBoundaries, List<Boundary> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
|
||||
return Stream.empty();
|
||||
}
|
||||
List<Boundary> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
|
||||
return entityBoundaries.stream()
|
||||
.map(boundary -> boundary.trim(node.getTextBlock()))
|
||||
.filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
private static List<Boundary> findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(List<Boundary> startBoundaries, List<Boundary> stopBoundaries) {
|
||||
|
||||
List<Boundary> entityBoundaries = new LinkedList<>();
|
||||
for (Boundary startBoundary : startBoundaries) {
|
||||
Optional<Boundary> optionalStopBoundaryWithMinimalDistance = stopBoundaries.stream()
|
||||
.filter(stopBoundary -> stopBoundary.start() > startBoundary.end())
|
||||
.min(Comparator.comparingInt(Boundary::start));
|
||||
if (optionalStopBoundaryWithMinimalDistance.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
entityBoundaries.add(new Boundary(startBoundary.end(), optionalStopBoundaryWithMinimalDistance.get().start()));
|
||||
}
|
||||
return removeOuterOverlappingBoundaries(entityBoundaries);
|
||||
}
|
||||
|
||||
|
||||
private static List<Boundary> removeOuterOverlappingBoundaries(List<Boundary> entityBoundaries) {
|
||||
/*
|
||||
In some cases we get boundaries, where one contains the other. This happens for Example when we have two start boundaries and one stop boundary after the two start boundaries.
|
||||
Then we get two boundaries where one is entirely contained in the other. So we want to remove the outer boundary.
|
||||
For Example consider the text: "a this is some text. a here is more text b". If "a" is the start string and "b" is the stop string, there are two possibilities.
|
||||
"this is some text. a here is more text" and "here is more text". We only want to keep the latter.
|
||||
*/
|
||||
return entityBoundaries.stream()
|
||||
.filter(boundary -> entityBoundaries.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary)))
|
||||
.toList();
|
||||
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
|
||||
.stream()
|
||||
.filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
|
||||
.map(bounds -> byBoundary(bounds, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.getTextBlock();
|
||||
SearchImplementation searchImplementation = new SearchImplementation(strings, false);
|
||||
return searchImplementation.getBoundaries(textBlock, node.getBoundary())
|
||||
.stream()
|
||||
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
|
||||
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.getTextBlock();
|
||||
return RedactionSearchUtility.findBoundariesByString(string, textBlock)
|
||||
.stream()
|
||||
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
|
||||
.filter(boundary -> isValidEntityBoundary(textBlock, boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
|
||||
|
||||
return tableNode.streamTableCells()
|
||||
.flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByString(string, tableCell.getTextBlock()),
|
||||
tableCell,
|
||||
type,
|
||||
entityType,
|
||||
tableNode));
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> lineAfterStringAcrossColumnsIgnoreCase(String string, String type, EntityType entityType, Table tableNode) {
|
||||
|
||||
return tableNode.streamTableCells()
|
||||
.flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, tableCell.getTextBlock()),
|
||||
tableCell,
|
||||
type,
|
||||
entityType,
|
||||
tableNode));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
|
||||
*
|
||||
* @param boundaries a list of boundaries
|
||||
* @param tableCell the table cell
|
||||
* @param type the type
|
||||
* @param entityType the entity type
|
||||
* @param tableNode the table node
|
||||
* @return a stream of RedactionEntities
|
||||
*/
|
||||
private Stream<RedactionEntity> lineAfterBoundariesAcrossColumns(List<Boundary> boundaries, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
|
||||
|
||||
return boundaries.stream()
|
||||
.map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
|
||||
.map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
|
||||
.map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
|
||||
.filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
|
||||
.map(nextTableCell -> RedactionSearchUtility.findBoundaryOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
|
||||
.map(b -> b.trim(tableNode.getTextBlock()))
|
||||
.filter(boundary -> isValidEntityBoundary(tableNode.getTextBlock(), boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, tableNode))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get))
|
||||
.flatMap(Functions.identity());
|
||||
}
|
||||
|
||||
|
||||
public Optional<RedactionEntity> semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) {
|
||||
|
||||
var textBlock = semanticNode.getTextBlock();
|
||||
int startIndex = Math.min(textBlock.indexOf(string), 0);
|
||||
var boundary = new Boundary(startIndex, semanticNode.getBoundary().end());
|
||||
if (boundary.length() > 0) {
|
||||
boundary = new Boundary(boundary.start(), boundary.end() - 1);
|
||||
}
|
||||
if (!isValidEntityBoundary(textBlock, boundary)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return byBoundary(boundary, type, entityType, semanticNode);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return byRegexWithLineBreaks(regexPattern, type, entityType, 0, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return byRegexWithLineBreaksIgnoreCase(regexPattern, type, entityType, 0, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return byRegex(regexPattern, type, entityType, 0, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return byRegexIgnoreCase(regexPattern, type, entityType, 0, node);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
|
||||
|
||||
return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
|
||||
|
||||
return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
|
||||
|
||||
return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
|
||||
|
||||
return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) {
|
||||
|
||||
return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> bySemanticNodeParagraphsOnlyMergeConsecutive(SemanticNode node, String type, EntityType entityType) {
|
||||
|
||||
return node.streamAllSubNodesOfType(NodeType.PARAGRAPH)
|
||||
.map(SemanticNode::getBoundary)
|
||||
.collect(new ConsecutiveBoundaryCollector())
|
||||
.stream()
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get);
|
||||
}
|
||||
|
||||
|
||||
public Optional<RedactionEntity> bySemanticNode(SemanticNode node, String type, EntityType entityType) {
|
||||
|
||||
Boundary boundary = node.getTextBlock().getBoundary();
|
||||
|
||||
if (boundary.length() > 0) {
|
||||
boundary = new Boundary(boundary.start(), boundary.end() - 1);
|
||||
}
|
||||
if (!isValidEntityBoundary(node.getTextBlock(), boundary)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return byBoundary(boundary, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public Optional<RedactionEntity> byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
|
||||
|
||||
int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
|
||||
return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
|
||||
}
|
||||
|
||||
|
||||
public Optional<RedactionEntity> bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
|
||||
|
||||
int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
|
||||
expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
|
||||
return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
|
||||
}
|
||||
|
||||
|
||||
private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
|
||||
|
||||
if (textBlock.getNextLinebreak(end) < expandedEnd) {
|
||||
return end;
|
||||
}
|
||||
return expandedEnd;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
|
||||
* If the document already contains an equal redaction entity, then the original Entity is returned.
|
||||
* Also inserts the Entity into the kieSession.
|
||||
*
|
||||
* @param boundary The boundary of the redaction entity.
|
||||
* @param type The type of the redaction entity.
|
||||
* @param entityType The entity type of the redaction entity.
|
||||
* @param node The semantic node to associate with the redaction entity.
|
||||
* @return An Optional containing the redaction entity, or the previous entity if the entity already exists.
|
||||
*/
|
||||
public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
if (!node.getBoundary().contains(boundary)) {
|
||||
throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", boundary, node.getBoundary(), node));
|
||||
}
|
||||
Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
|
||||
RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
|
||||
if (node.getEntities().contains(entity)) {
|
||||
return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny();
|
||||
}
|
||||
addEntityToGraph(entity, node);
|
||||
entity.addEngine(Engine.RULE);
|
||||
insertToKieSession(entity);
|
||||
return Optional.of(entity);
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity forceByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
|
||||
RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
|
||||
addEntityToGraph(entity, node);
|
||||
return entity;
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byEntities(List<RedactionEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
|
||||
throw new IllegalArgumentException("Provided entities can not be merged, since they do not intersect or are not the same type!" + entitiesToMerge);
|
||||
}
|
||||
if (entitiesToMerge.isEmpty()) {
|
||||
throw new IllegalArgumentException("No Entities to merge!");
|
||||
}
|
||||
if (entitiesToMerge.size() == 1) {
|
||||
return entitiesToMerge.get(0);
|
||||
}
|
||||
|
||||
RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
|
||||
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
|
||||
entitiesToMerge.stream().map(RedactionEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
|
||||
|
||||
mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry));
|
||||
mergedEntity.setDossierDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDossierDictionaryEntry));
|
||||
mergedEntity.setIgnored(entitiesToMerge.stream().allMatch(RedactionEntity::isIgnored));
|
||||
mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
|
||||
|
||||
addEntityToGraph(mergedEntity, node);
|
||||
insertToKieSession(mergedEntity);
|
||||
return mergedEntity;
|
||||
}
|
||||
|
||||
|
||||
private void insertToKieSession(RedactionEntity mergedEntity) {
|
||||
|
||||
if (kieSession != null) {
|
||||
kieSession.insert(mergedEntity);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
|
||||
|
||||
var entity = forceByBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
|
||||
entity.addEngine(Engine.NER);
|
||||
insertToKieSession(entity);
|
||||
return entity;
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
|
||||
|
||||
var entity = forceByBoundary(nerEntity.boundary(), type, entityType, semanticNode);
|
||||
entity.addEngine(Engine.NER);
|
||||
insertToKieSession(entity);
|
||||
return entity;
|
||||
}
|
||||
|
||||
|
||||
public Stream<RedactionEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
|
||||
|
||||
return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
|
||||
.map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
|
||||
.peek(entity -> entity.addEngine(Engine.NER))
|
||||
.peek(this::insertToKieSession);
|
||||
}
|
||||
|
||||
|
||||
public RedactionEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
|
||||
|
||||
RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCell.getBoundary().start(), tableCell.getBoundary().start()), type, entityType);
|
||||
|
||||
String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
|
||||
highlightEntity.setRedactionPositionsPerPage(tableCell.getBBox()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(entry -> new RedactionPosition(positionId, entry.getKey(), List.of(entry.getValue())))
|
||||
.toList());
|
||||
addEntityToGraph(highlightEntity, tableCell);
|
||||
return highlightEntity;
|
||||
}
|
||||
|
||||
|
||||
public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
|
||||
|
||||
return boundary.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, boundary);
|
||||
}
|
||||
|
||||
|
||||
public void addEntityToGraph(RedactionEntity entity, SemanticNode node) {
|
||||
|
||||
DocumentTree documentTree = node.getDocumentTree();
|
||||
try {
|
||||
addEntityToGraph(entity, documentTree);
|
||||
entity.addIntersectingNode(documentTree.getRoot().getNode());
|
||||
documentTree.getRoot().getNode().getEntities().add(entity);
|
||||
} catch (NoSuchElementException e) {
|
||||
entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
|
||||
entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
|
||||
entity.addIntersectingNode(documentTree.getRoot().getNode());
|
||||
addToPages(entity);
|
||||
addEntityToNodeEntitySets(entity);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addEntityToGraph(RedactionEntity entity, DocumentTree documentTree) {
|
||||
|
||||
SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
|
||||
.filter(node -> node.getTextBlock().containsBoundary(entity.getBoundary()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
|
||||
|
||||
containingNode.addThisToEntityIfIntersects(entity);
|
||||
|
||||
TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
|
||||
entityEnrichmentService.enrichEntity(entity, textBlock);
|
||||
|
||||
addToPages(entity);
|
||||
addEntityToNodeEntitySets(entity);
|
||||
}
|
||||
|
||||
|
||||
private static void addToPages(RedactionEntity entity) {
|
||||
|
||||
Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary());
|
||||
entity.getPages().addAll(pages);
|
||||
pages.forEach(page -> page.getEntities().add(entity));
|
||||
}
|
||||
|
||||
|
||||
private static void addEntityToNodeEntitySets(RedactionEntity entity) {
|
||||
|
||||
entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
|
||||
}
|
||||
|
||||
|
||||
private static boolean allEntitiesIntersectAndHaveSameTypes(List<RedactionEntity> entitiesToMerge) {
|
||||
|
||||
if (entitiesToMerge.isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
RedactionEntity previousEntity = entitiesToMerge.get(0);
|
||||
for (RedactionEntity redactionEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
|
||||
boolean typeMatches = redactionEntity.getType().equals(previousEntity.getType());
|
||||
boolean entityTypeMatches = redactionEntity.getEntityType().equals(previousEntity.getEntityType());
|
||||
boolean intersects = redactionEntity.intersects(previousEntity);
|
||||
if (!typeMatches || !entityTypeMatches || !intersects) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
|
||||
|
||||
return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())).trim(textBlock);
|
||||
}
|
||||
|
||||
}
|
||||
+10
-10
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service.document;
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
@@ -6,9 +6,9 @@ import java.util.Objects;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@@ -19,17 +19,17 @@ public class EntityEnrichmentService {
|
||||
private final RedactionServiceSettings redactionServiceSettings;
|
||||
|
||||
|
||||
public void enrichEntity(TextEntity entity, TextBlock textBlock) {
|
||||
public void enrichEntity(RedactionEntity entity, TextBlock textBlock) {
|
||||
|
||||
entity.setValue(textBlock.subSequence(entity.getTextRange()).toString());
|
||||
entity.setTextAfter(findTextAfter(entity.getTextRange().end(), textBlock));
|
||||
entity.setTextBefore(findTextBefore(entity.getTextRange().start(), textBlock));
|
||||
entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
|
||||
entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
|
||||
entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
|
||||
}
|
||||
|
||||
|
||||
private String findTextAfter(int index, TextBlock textBlock) {
|
||||
|
||||
int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().end());
|
||||
int endOffset = Math.min(index + redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().end());
|
||||
String textAfter = textBlock.subSequence(index, endOffset).toString();
|
||||
if (!textAfter.isBlank()) {
|
||||
List<String> wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter);
|
||||
@@ -44,7 +44,7 @@ public class EntityEnrichmentService {
|
||||
|
||||
private String findTextBefore(int index, TextBlock textBlock) {
|
||||
|
||||
int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getTextRange().start());
|
||||
int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start());
|
||||
String textBefore = textBlock.subSequence(offsetBefore, index).toString();
|
||||
if (!textBefore.isBlank()) {
|
||||
List<String> wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore);
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.document.utils.RectangleTransformations;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
public class ManualRedactionApplicationService {
|
||||
|
||||
private final EntityCreationService entityCreationService;
|
||||
|
||||
|
||||
public void resizeEntityAndReinsert(RedactionEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
|
||||
|
||||
RedactionPosition redactionPositionToBeResized = entityToBeResized.getRedactionPositionsPerPage()
|
||||
.stream()
|
||||
.filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
|
||||
|
||||
redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
|
||||
|
||||
int newStartOffset;
|
||||
if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
|
||||
newStartOffset = entityToBeResized.getBoundary().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
|
||||
} else {
|
||||
newStartOffset = entityToBeResized.getBoundary().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
|
||||
}
|
||||
|
||||
SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
|
||||
entityToBeResized.removeFromGraph();
|
||||
entityToBeResized.setResized(true);
|
||||
entityToBeResized.setRemoved(false);
|
||||
entityToBeResized.setIgnored(false);
|
||||
entityToBeResized.getBoundary().setStart(newStartOffset);
|
||||
entityToBeResized.getBoundary().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
|
||||
entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
|
||||
}
|
||||
|
||||
|
||||
public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
|
||||
|
||||
if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {
|
||||
return;
|
||||
}
|
||||
var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
|
||||
image.setPosition(bBox);
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
|
||||
|
||||
return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
|
||||
}
|
||||
|
||||
}
|
||||
+26
-11
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils;
|
||||
package com.iqser.red.service.redaction.v1.server.document.utils;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.awt.geom.RectangularShape;
|
||||
@@ -12,23 +12,44 @@ import java.util.function.Function;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collector;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
public class RectangleTransformations {
|
||||
|
||||
|
||||
public static Rectangle2D atomicTextBlockBBox(List<AtomicTextBlock> atomicTextBlocks) {
|
||||
|
||||
return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DBBoxCollector());
|
||||
}
|
||||
|
||||
|
||||
public static Rectangle2D rectangleBBox(List<Position> positions) {
|
||||
public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {
|
||||
|
||||
return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector());
|
||||
return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
|
||||
}
|
||||
|
||||
|
||||
public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
|
||||
|
||||
return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
|
||||
redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(),
|
||||
redactionLogRectangle.getWidth(),
|
||||
-redactionLogRectangle.getHeight());
|
||||
}
|
||||
|
||||
|
||||
public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
|
||||
|
||||
return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
|
||||
(float) rectangle2D.getWidth(),
|
||||
-(float) rectangle2D.getHeight(),
|
||||
pageNumber);
|
||||
}
|
||||
|
||||
|
||||
@@ -70,12 +91,6 @@ public class RectangleTransformations {
|
||||
}
|
||||
|
||||
|
||||
public static Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> collectBBox() {
|
||||
|
||||
return new Rectangle2DBBoxCollector();
|
||||
}
|
||||
|
||||
|
||||
private static class Rectangle2DBBoxCollector implements Collector<Rectangle2D, Rectangle2DBBoxCollector.BBox, Rectangle2D> {
|
||||
|
||||
@Override
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.utils;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
@UtilityClass
|
||||
public class RedactionSearchUtility {
|
||||
|
||||
public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
|
||||
return pattern.matcher(charSequence).find();
|
||||
}
|
||||
|
||||
|
||||
public static boolean anyMatchIgnoreCase(CharSequence charSequence, String regexPattern) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
|
||||
return pattern.matcher(charSequence).find();
|
||||
}
|
||||
|
||||
|
||||
public static boolean exactMatch(CharSequence charSequence, String regexPattern) {
|
||||
|
||||
return charSequence.toString().matches(regexPattern);
|
||||
}
|
||||
|
||||
|
||||
public static boolean anyMatchIgnoreCase(TextBlock textBlock, String regexPattern) {
|
||||
|
||||
return anyMatchIgnoreCase(textBlock.getSearchText(), regexPattern);
|
||||
}
|
||||
|
||||
|
||||
public static boolean anyMatch(TextBlock textBlock, String regexPattern) {
|
||||
|
||||
return anyMatch(textBlock.getSearchText(), regexPattern);
|
||||
}
|
||||
|
||||
|
||||
public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
|
||||
Matcher matcher = pattern.matcher(searchText);
|
||||
if (matcher.find()) {
|
||||
return new Boundary(matcher.start(), matcher.end());
|
||||
}
|
||||
throw new IllegalArgumentException(format("Charsequence %s does not contain any matches for pattern %s", searchText, regexPattern));
|
||||
}
|
||||
|
||||
|
||||
public static int getExpandedEndByRegex(RedactionEntity entity, String regexPattern) {
|
||||
|
||||
int expandedEnd;
|
||||
if (anyMatch(entity.getTextAfter(), regexPattern)) {
|
||||
Boundary postfixBoundary = findFirstBoundary(regexPattern, entity.getTextAfter());
|
||||
expandedEnd = postfixBoundary.end() + entity.getBoundary().end();
|
||||
} else {
|
||||
expandedEnd = entity.getBoundary().end();
|
||||
}
|
||||
return expandedEnd;
|
||||
}
|
||||
|
||||
|
||||
public static int getExpandedStartByRegex(RedactionEntity entity, String regexPattern) {
|
||||
|
||||
int expandedStart;
|
||||
if (anyMatch(entity.getTextBefore(), regexPattern)) {
|
||||
Boundary prefixBoundary = findFirstBoundary(regexPattern, entity.getTextBefore());
|
||||
expandedStart = prefixBoundary.start() + entity.getBoundary().start() - entity.getTextBefore().length();
|
||||
} else {
|
||||
expandedStart = entity.getBoundary().start();
|
||||
}
|
||||
return expandedStart;
|
||||
}
|
||||
|
||||
public static Boundary findBoundaryOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {
|
||||
|
||||
List<Boundary> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed().map(textBlock::getLineBoundary).filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary)).toList();
|
||||
if (lineBoundaries.isEmpty()) {
|
||||
return new Boundary(textBlock.getBoundary().start(), textBlock.getBoundary().start());
|
||||
}
|
||||
return Boundary.merge(lineBoundaries);
|
||||
}
|
||||
|
||||
|
||||
private static boolean isWithinYRange(double maxY, double minY, TextBlock textBlock, Boundary lineBoundary) {
|
||||
|
||||
Rectangle2D lineBBox = RectangleTransformations.rectangle2DBBox(textBlock.getPositions(lineBoundary));
|
||||
return lineBBox.getMinY() < maxY && minY < lineBBox.getMaxY();
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
|
||||
return getBoundariesByPattern(textBlock, 0, pattern);
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByRegex(String regexPattern, int group, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
|
||||
return getBoundariesByPattern(textBlock, group, pattern);
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByRegexWithLineBreaks(String regexPattern, int group, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, false);
|
||||
return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern);
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByRegexWithLineBreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, true);
|
||||
return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern);
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
|
||||
return getBoundariesByPattern(textBlock, group, pattern);
|
||||
}
|
||||
|
||||
|
||||
private static List<Boundary> getBoundariesByPattern(TextBlock textBlock, int group, Pattern pattern) {
|
||||
|
||||
Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
|
||||
List<Boundary> boundaries = new LinkedList<>();
|
||||
while (matcher.find()) {
|
||||
boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start()));
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
|
||||
private static List<Boundary> getBoundariesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) {
|
||||
|
||||
String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks();
|
||||
Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
|
||||
List<Boundary> boundaries = new LinkedList<>();
|
||||
while (matcher.find()) {
|
||||
boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start()));
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByString(String searchString, TextBlock textBlock) {
|
||||
|
||||
List<Boundary> boundaries = new LinkedList<>();
|
||||
for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) {
|
||||
boundaries.add(new Boundary(index, index + searchString.length()));
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByStringIgnoreCase(String searchString, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Pattern.compile(Pattern.quote(searchString), Pattern.CASE_INSENSITIVE);
|
||||
return getBoundariesByPattern(textBlock, 0, pattern);
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils.exception;
|
||||
package com.iqser.red.service.redaction.v1.server.exception;
|
||||
|
||||
public class NotFoundException extends RuntimeException {
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils.exception;
|
||||
package com.iqser.red.service.redaction.v1.server.exception;
|
||||
|
||||
public class RedactionException extends RuntimeException {
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.utils.exception;
|
||||
package com.iqser.red.service.redaction.v1.server.exception;
|
||||
|
||||
public class RulesValidationException extends RuntimeException {
|
||||
|
||||
-428
@@ -1,428 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.migration;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class LegacyRedactionLogMergeService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
|
||||
public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
|
||||
|
||||
Set<String> skippedImportedRedactions = new HashSet<>();
|
||||
log.info("Adding manual add Entries and removing skipped or imported entries");
|
||||
if (manualRedactions != null) {
|
||||
|
||||
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
|
||||
|
||||
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
|
||||
|
||||
var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
|
||||
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
|
||||
if (entry.isImported()) {
|
||||
processRedactionLogEntry(manualRedactionWrappers.stream()
|
||||
.filter(ManualRedactionWrapper::isApproved)
|
||||
.filter(mr -> entry.getId().equals(mr.getId()))
|
||||
.collect(Collectors.toList()), entry, dossierTemplateId);
|
||||
if (!entry.isRedacted()) {
|
||||
skippedImportedRedactions.add(entry.getId());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Set<String> processedIds = new HashSet<>();
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> {
|
||||
if (entry.isFalsePositive()) {
|
||||
return true;
|
||||
}
|
||||
if (entry.getImportedRedactionIntersections() != null) {
|
||||
entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
|
||||
if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
|
||||
.equals("ocr")))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (processedIds.contains(entry.getId())) {
|
||||
log.info("Duplicate annotation found with id {}", entry.getId());
|
||||
return true;
|
||||
}
|
||||
processedIds.add(entry.getId());
|
||||
return false;
|
||||
});
|
||||
return redactionLog;
|
||||
}
|
||||
|
||||
public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
|
||||
|
||||
return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
|
||||
}
|
||||
|
||||
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
|
||||
|
||||
List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
|
||||
|
||||
manualRedactions.getRecategorizations()
|
||||
.forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getIdsToRemove()
|
||||
.forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getForceRedactions()
|
||||
.forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getLegalBasisChanges()
|
||||
.forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
|
||||
}
|
||||
});
|
||||
|
||||
manualRedactions.getResizeRedactions()
|
||||
.forEach(item -> {
|
||||
if (item.getSoftDeletedTime() == null) {
|
||||
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
|
||||
}
|
||||
});
|
||||
|
||||
Collections.sort(manualRedactionWrappers);
|
||||
|
||||
return manualRedactionWrappers;
|
||||
}
|
||||
|
||||
|
||||
private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
|
||||
|
||||
manualRedactionWrappers.forEach(mrw -> {
|
||||
|
||||
Object item = mrw.getItem();
|
||||
if (item instanceof ManualRecategorization imageRecategorization) {
|
||||
processManualImageRecategorization(redactionLogEntry, dossierTemplateId, imageRecategorization);
|
||||
}
|
||||
|
||||
if (item instanceof IdRemoval manualRemoval) {
|
||||
processIdRemoval(redactionLogEntry, manualRemoval);
|
||||
}
|
||||
|
||||
if (item instanceof ManualForceRedaction manualForceRedact) {
|
||||
processManualForceRedaction(redactionLogEntry, dossierTemplateId, manualForceRedact);
|
||||
}
|
||||
|
||||
if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
|
||||
processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
|
||||
}
|
||||
|
||||
if (item instanceof ManualResizeRedaction manualResizeRedact) {
|
||||
processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualRecategorization imageRecategorization) {
|
||||
|
||||
String manualOverrideReason = null;
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
|
||||
redactionLogEntry.setType(imageRecategorization.getType());
|
||||
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
|
||||
|
||||
if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setHint(true);
|
||||
} else {
|
||||
redactionLogEntry.setHint(false);
|
||||
}
|
||||
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
|
||||
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
|
||||
}
|
||||
|
||||
|
||||
private String mergeReasonIfNecessary(String currentReason, String addition) {
|
||||
|
||||
if (currentReason != null) {
|
||||
if (!currentReason.contains(addition)) {
|
||||
return currentReason + addition;
|
||||
}
|
||||
return currentReason;
|
||||
} else {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
|
||||
|
||||
boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
|
||||
if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
|
||||
log.debug("Skipping merge for dictionary-modifying entry");
|
||||
} else {
|
||||
String manualOverrideReason = null;
|
||||
if (isApprovedRedaction) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
|
||||
redactionLogEntry.setHint(false);
|
||||
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
}
|
||||
|
||||
|
||||
private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
|
||||
}
|
||||
|
||||
|
||||
private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualForceRedaction manualForceRedact) {
|
||||
|
||||
String manualOverrideReason = null;
|
||||
var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
|
||||
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
// Forcing a skipped hint should result in a hint
|
||||
if (dictionaryIsHint) {
|
||||
redactionLogEntry.setHint(true);
|
||||
} else {
|
||||
redactionLogEntry.setRedacted(true);
|
||||
}
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
}
|
||||
|
||||
|
||||
private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
|
||||
|
||||
String manualOverrideReason = null;
|
||||
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
|
||||
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
redactionLogEntry.setRedacted(true);
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
|
||||
}
|
||||
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
|
||||
redactionLogEntry.setValue(manualLegalBasisChange.getValue());
|
||||
}
|
||||
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
|
||||
}
|
||||
|
||||
if (manualOverrideReason != null) {
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
}
|
||||
|
||||
var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
|
||||
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
manualChange.withChange("section", manualLegalBasisChange.getSection());
|
||||
}
|
||||
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
|
||||
manualChange.withChange("value", manualLegalBasisChange.getValue());
|
||||
}
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
}
|
||||
|
||||
|
||||
private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
|
||||
|
||||
String manualOverrideReason = null;
|
||||
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
|
||||
redactionLogEntry.setValue(manualResizeRedact.getValue());
|
||||
}
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
|
||||
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
|
||||
// This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
|
||||
if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
}
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.getManualChanges()
|
||||
.add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
|
||||
|
||||
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
|
||||
|
||||
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
|
||||
|
||||
if (shouldCreateManualEntry(manualRedactionEntry)) {
|
||||
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
|
||||
redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
|
||||
redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
|
||||
|
||||
redactionLogEntries.add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
return redactionLogEntries;
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
|
||||
|
||||
return positions.stream()
|
||||
.map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("PMD.UselessParentheses")
|
||||
private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
if (!manualRedactionEntry.isApproved()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
|
||||
|| manualRedactionEntry.isAddToDossierDictionary())
|
||||
&& manualRedactionEntry.getProcessedDate() == null);
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
|
||||
|
||||
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
|
||||
|
||||
var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
|
||||
List<ManualChange> changeList = new ArrayList<>();
|
||||
changeList.add(change);
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
|
||||
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.sourceId(manualRedactionEntry.getSourceId())
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.sectionNumber(-1)
|
||||
.rectangle(manualRedactionEntry.isRectangle())
|
||||
.manualChanges(changeList)
|
||||
.changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
|
||||
|
||||
private String id;
|
||||
private OffsetDateTime date;
|
||||
private Object item;
|
||||
private boolean approved;
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(ManualRedactionWrapper o) {
|
||||
|
||||
return this.date.compareTo(o.date);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
-70
@@ -1,70 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.migration;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Service
|
||||
@SuppressWarnings("PMD")
|
||||
public class LegacyVersion0MigrationService {
|
||||
|
||||
public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
|
||||
|
||||
List<RedactionLogEntry> mergedEntries = new LinkedList<>();
|
||||
Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId));
|
||||
for (List<RedactionLogEntry> entries : entriesById.values()) {
|
||||
|
||||
if (entries.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entries.size() == 1) {
|
||||
mergedEntries.add(entries.get(0));
|
||||
continue;
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList();
|
||||
|
||||
RedactionLogEntry initialEntry = sortedEntries.get(0);
|
||||
for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {
|
||||
copyNonNullFields(entry, initialEntry);
|
||||
}
|
||||
mergedEntries.add(initialEntry);
|
||||
}
|
||||
|
||||
redactionLog.setRedactionLogEntry(mergedEntries);
|
||||
return redactionLog;
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public static void copyNonNullFields(RedactionLogEntry source, RedactionLogEntry destination) {
|
||||
|
||||
if (source == null || destination == null) {
|
||||
throw new IllegalArgumentException("Source and destination objects must not be null");
|
||||
}
|
||||
|
||||
Class<?> sourceClass = source.getClass();
|
||||
|
||||
Field[] sourceFields = sourceClass.getDeclaredFields();
|
||||
|
||||
for (Field field : sourceFields) {
|
||||
field.setAccessible(true);
|
||||
Object value = field.get(source);
|
||||
if (value != null) {
|
||||
field.set(destination, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.migration;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
|
||||
public class MigrationMapper {
|
||||
|
||||
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
|
||||
|
||||
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
|
||||
toEntityLogType(change.getType()),
|
||||
change.getDateTime());
|
||||
}
|
||||
|
||||
|
||||
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
|
||||
|
||||
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
|
||||
manualChange.getProcessedDate(),
|
||||
manualChange.getRequestedDate(),
|
||||
manualChange.getUserId(),
|
||||
manualChange.getPropertyChanges());
|
||||
}
|
||||
|
||||
|
||||
public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
|
||||
|
||||
return switch (type) {
|
||||
case ADDED -> ChangeType.ADDED;
|
||||
case REMOVED -> ChangeType.REMOVED;
|
||||
case CHANGED -> ChangeType.CHANGED;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
|
||||
|
||||
return switch (manualRedactionType) {
|
||||
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
|
||||
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
|
||||
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
|
||||
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
|
||||
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
|
||||
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
|
||||
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
|
||||
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
|
||||
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
|
||||
|
||||
return switch (engine) {
|
||||
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
|
||||
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
|
||||
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
|
||||
|
||||
if (entry.getEngines() == null) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
return entry.getEngines()
|
||||
.stream()
|
||||
.map(MigrationMapper::toEntityLogEngine)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
|
||||
|
||||
if (manualChanges == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return manualChanges.stream()
|
||||
.map(MigrationMapper::toEntityLogManualChanges)
|
||||
.toList();
|
||||
}
|
||||
|
||||
}
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.migration;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
import org.springframework.amqp.rabbit.core.RabbitTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.MigrationResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class MigrationMessageReceiver {
|
||||
|
||||
ObjectMapper objectMapper;
|
||||
RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
|
||||
RedactionStorageService redactionStorageService;
|
||||
LegacyRedactionLogMergeService legacyRedactionLogMergeService;
|
||||
LegacyVersion0MigrationService legacyVersion0MigrationService;
|
||||
RabbitTemplate rabbitTemplate;
|
||||
DictionaryService dictionaryService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = MIGRATION_QUEUE)
|
||||
public void receiveMigrationRequest(Message message) {
|
||||
|
||||
MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
|
||||
log.info("--------------------------------------------------------------------");
|
||||
log.info("Starting redactionLog to entityLog migration for dossierId {} and fileId {}", migrationRequest.getDossierId(), migrationRequest.getFileId());
|
||||
|
||||
dictionaryService.updateDictionary(migrationRequest.getDossierTemplateId(), migrationRequest.getDossierId());
|
||||
|
||||
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
|
||||
RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
|
||||
|
||||
if (redactionLog.getAnalysisVersion() == 0) {
|
||||
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
|
||||
} else if (migrationRequest.getManualRedactions() != null) {
|
||||
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
|
||||
migrationRequest.getManualRedactions(),
|
||||
migrationRequest.getDossierTemplateId());
|
||||
}
|
||||
|
||||
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
|
||||
document,
|
||||
migrationRequest.getDossierTemplateId(),
|
||||
migrationRequest.getManualRedactions(),
|
||||
migrationRequest.getFileId());
|
||||
|
||||
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
|
||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
|
||||
|
||||
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
|
||||
log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
|
||||
migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
|
||||
migratedEntityLog.getMigratedIds().getMappings().size(),
|
||||
migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
|
||||
migrationRequest.getDossierId(),
|
||||
migrationRequest.getFileId());
|
||||
log.info("");
|
||||
}
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
public void sendFinished(MigrationResponse migrationResponse) {
|
||||
|
||||
rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
|
||||
}
|
||||
|
||||
}
|
||||
-290
@@ -1,290 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.migration;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
||||
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
|
||||
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
//TODO: remove this, once the migration is done
|
||||
public class RedactionLogToEntityLogMigrationService {
|
||||
|
||||
private static final double MATCH_THRESHOLD = 10;
|
||||
EntityFindingUtility entityFindingUtility;
|
||||
EntityEnrichmentService entityEnrichmentService;
|
||||
DictionaryService dictionaryService;
|
||||
ManualChangesApplicationService manualChangesApplicationService;
|
||||
|
||||
|
||||
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
|
||||
|
||||
log.info("Migrating entities for file {}", fileId);
|
||||
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
|
||||
|
||||
MigratedIds migratedIds = entitiesToMigrate.stream()
|
||||
.collect(new MigratedIdsCollector());
|
||||
|
||||
applyManualChanges(entitiesToMigrate, manualRedactions);
|
||||
log.info("applying manual changes to migrated entities for file {}", fileId);
|
||||
|
||||
EntityLog entityLog = new EntityLog();
|
||||
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
||||
entityLog.setRulesVersion(redactionLog.getRulesVersion());
|
||||
entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
|
||||
entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
|
||||
entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
|
||||
entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
|
||||
entityLog.setLegalBasis(redactionLog.getLegalBasis()
|
||||
.stream()
|
||||
.map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
|
||||
.toList());
|
||||
|
||||
Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
|
||||
|
||||
log.info("Writing migrated entities to entityLog for file {}", fileId);
|
||||
entityLog.setEntityLogEntry(entitiesToMigrate.stream()
|
||||
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
|
||||
.toList());
|
||||
|
||||
if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
|
||||
String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
|
||||
redactionLog.getRedactionLogEntry().size(),
|
||||
entityLog.getEntityLogEntry().size());
|
||||
log.error(message);
|
||||
throw new AssertionError(message);
|
||||
}
|
||||
|
||||
MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
|
||||
.filter(MigrationEntity::hasManualChangesOrComments)
|
||||
.filter(m -> !m.getOldId().equals(m.getNewId()))
|
||||
.collect(new MigratedIdsCollector());
|
||||
|
||||
List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
|
||||
.filter(MigrationEntity::needsManualEntry)
|
||||
.map(MigrationEntity::buildManualRedactionEntry)
|
||||
.toList();
|
||||
|
||||
idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
|
||||
|
||||
return new MigratedEntityLog(idsToMigrateInDb, entityLog);
|
||||
}
|
||||
|
||||
|
||||
private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
|
||||
manualRedactions.getEntriesToAdd(),
|
||||
manualRedactions.getForceRedactions(),
|
||||
manualRedactions.getResizeRedactions(),
|
||||
manualRedactions.getLegalBasisChanges(),
|
||||
manualRedactions.getRecategorizations())
|
||||
.flatMap(Collection::stream)
|
||||
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
||||
|
||||
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
|
||||
Collections.emptyList()),
|
||||
manualChangesApplicationService));
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
|
||||
|
||||
return redactionLog.getRedactionLogEntry().size();
|
||||
}
|
||||
|
||||
|
||||
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||
|
||||
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
|
||||
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
|
||||
return Stream.of(textMigrationEntities.stream(), images.stream())
|
||||
.flatMap(Function.identity())
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
|
||||
|
||||
return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
|
||||
}
|
||||
|
||||
|
||||
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
|
||||
|
||||
List<Image> images = document.streamAllImages()
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(RedactionLogEntry::isImage)
|
||||
.toList();
|
||||
|
||||
List<MigrationEntity> migrationEntities = new LinkedList<>();
|
||||
for (RedactionLogEntry redactionLogImage : redactionLogImages) {
|
||||
List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromRedactionLogRectangle)
|
||||
.toList();
|
||||
assert imagePositions.size() == 1;
|
||||
Optional<Image> optionalClosestImage = images.stream()
|
||||
.filter(image -> image.onPage(redactionLogImage.getPositions()
|
||||
.get(0).getPage()))
|
||||
.min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
|
||||
.filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
|
||||
|
||||
Image closestImage;
|
||||
if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
|
||||
closestImage = buildImageDirectly(document, redactionLogImage);
|
||||
} else {
|
||||
closestImage = optionalClosestImage.get();
|
||||
images.remove(closestImage);
|
||||
}
|
||||
|
||||
String ruleIdentifier;
|
||||
String reason = Optional.ofNullable(redactionLogImage.getReason())
|
||||
.orElse("");
|
||||
if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
|
||||
ruleIdentifier = "OLDIMG.0.0";
|
||||
} else {
|
||||
ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
|
||||
}
|
||||
|
||||
if (redactionLogImage.lastChangeIsRemoved()) {
|
||||
closestImage.remove(ruleIdentifier, reason);
|
||||
} else if (redactionLogImage.isRedacted()) {
|
||||
closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
|
||||
} else {
|
||||
closestImage.skip(ruleIdentifier, reason);
|
||||
}
|
||||
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
|
||||
}
|
||||
return migrationEntities;
|
||||
}
|
||||
|
||||
|
||||
private static Image buildImageDirectly(Document document, RedactionLogEntry redactionLogImage) {
|
||||
|
||||
Image image = Image.builder()
|
||||
.documentTree(document.getDocumentTree())
|
||||
.imageType(ImageType.fromString(redactionLogImage.getType()))
|
||||
.transparent(redactionLogImage.isImageHasTransparency())
|
||||
.page(document.getPages()
|
||||
.stream()
|
||||
.filter(p -> p.getNumber() == redactionLogImage.getPositions()
|
||||
.get(0).getPage())
|
||||
.findFirst()
|
||||
.orElseThrow())
|
||||
.position(toRectangle2D(redactionLogImage.getPositions()
|
||||
.get(0)))
|
||||
.build();
|
||||
|
||||
List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
|
||||
image.setTreeId(treeId);
|
||||
image.setId(IdBuilder.buildId(image.getPages(),
|
||||
image.getBBox().values()
|
||||
.stream()
|
||||
.toList(),
|
||||
"",
|
||||
""));
|
||||
|
||||
return image;
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(Rectangle rect) {
|
||||
|
||||
return new Rectangle2D.Double(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
|
||||
}
|
||||
|
||||
|
||||
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||
|
||||
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
||||
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
|
||||
.toList();
|
||||
|
||||
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
|
||||
.map(MigrationEntity::getPrecursorEntity)
|
||||
.toList();
|
||||
|
||||
log.info("Finding all possible entities");
|
||||
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
|
||||
|
||||
for (MigrationEntity migrationEntity : entitiesToMigrate) {
|
||||
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
|
||||
tempEntitiesByValue,
|
||||
MATCH_THRESHOLD);
|
||||
|
||||
if (optionalTextEntity.isEmpty()) {
|
||||
migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
|
||||
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
|
||||
migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
|
||||
continue;
|
||||
}
|
||||
|
||||
TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
|
||||
|
||||
migrationEntity.setMigratedEntity(migratedEntity);
|
||||
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
|
||||
migrationEntity.setNewId(migratedEntity.getId());
|
||||
}
|
||||
|
||||
tempEntitiesByValue.values()
|
||||
.stream()
|
||||
.flatMap(Collection::stream)
|
||||
.forEach(TextEntity::removeFromGraph);
|
||||
|
||||
return entitiesToMigrate;
|
||||
}
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
@Builder
|
||||
@Getter
|
||||
@Setter
|
||||
public class ClosestEntity {
|
||||
|
||||
private double distance;
|
||||
private TextEntity textEntity;
|
||||
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
|
||||
public record KieWrapper(KieContainer container, long rulesVersion) {
|
||||
|
||||
public static KieWrapper empty() {
|
||||
|
||||
return new KieWrapper(null, -1);
|
||||
}
|
||||
|
||||
|
||||
public boolean isPresent() {
|
||||
|
||||
return container != null && rulesVersion >= 0;
|
||||
}
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class MigratedEntityLog {
|
||||
|
||||
MigratedIds migratedIds;
|
||||
EntityLog entityLog;
|
||||
}
|
||||
-441
@@ -1,441 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
|
||||
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public final class MigrationEntity {
|
||||
|
||||
private final PrecursorEntity precursorEntity;
|
||||
private final RedactionLogEntry redactionLogEntry;
|
||||
private IEntity migratedEntity;
|
||||
private String oldId;
|
||||
private String newId;
|
||||
private String fileId;
|
||||
|
||||
@Builder.Default
|
||||
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
||||
|
||||
|
||||
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
|
||||
|
||||
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
|
||||
|
||||
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
|
||||
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
} else if (redactionLogEntry.lastChangeIsRemoved()) {
|
||||
precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
} else if (lastManualChangeIsRemove(redactionLogEntry)) {
|
||||
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
} else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
|
||||
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
} else if (precursorEntity.isApplied()) {
|
||||
precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
|
||||
} else {
|
||||
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
}
|
||||
|
||||
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
|
||||
}
|
||||
|
||||
|
||||
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
|
||||
|
||||
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
|
||||
}
|
||||
|
||||
|
||||
private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.getManualChanges() == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return redactionLogEntry.getManualChanges()
|
||||
.stream()
|
||||
.reduce((a, b) -> b)
|
||||
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
|
||||
public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
|
||||
|
||||
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
|
||||
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromRedactionLogRectangle)
|
||||
.toList();
|
||||
EntityType entityType = getEntityType(redactionLogEntry, hint);
|
||||
return PrecursorEntity.builder()
|
||||
.id(redactionLogEntry.getId())
|
||||
.value(redactionLogEntry.getValue())
|
||||
.entityPosition(rectangleWithPages)
|
||||
.ruleIdentifier(ruleIdentifier)
|
||||
.reason(Optional.ofNullable(redactionLogEntry.getReason())
|
||||
.orElse(""))
|
||||
.legalBasis(redactionLogEntry.getLegalBasis())
|
||||
.type(redactionLogEntry.getType())
|
||||
.section(redactionLogEntry.getSection())
|
||||
.engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
|
||||
.entityType(entityType)
|
||||
.applied(redactionLogEntry.isRedacted())
|
||||
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
|
||||
.isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
|
||||
.rectangle(redactionLogEntry.isRectangle())
|
||||
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static String buildRuleIdentifier(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
String ruleIdentifier;
|
||||
if (redactionLogEntry.getMatchedRule() != null) {
|
||||
ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
|
||||
} else {
|
||||
ruleIdentifier = "MAN.5.0"; // pure ManualRedactions used to have no matched rule
|
||||
}
|
||||
return ruleIdentifier;
|
||||
}
|
||||
|
||||
|
||||
private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
|
||||
|
||||
if (hint) {
|
||||
return EntityType.HINT;
|
||||
}
|
||||
if (redactionLogEntry.isFalsePositive()) {
|
||||
return EntityType.FALSE_POSITIVE;
|
||||
}
|
||||
if (redactionLogEntry.isHint()) {
|
||||
return EntityType.HINT;
|
||||
}
|
||||
if (redactionLogEntry.isRecommendation()) {
|
||||
return EntityType.RECOMMENDATION;
|
||||
}
|
||||
return EntityType.ENTITY;
|
||||
}
|
||||
|
||||
|
||||
private static EntryType getEntryType(EntityType entityType) {
|
||||
|
||||
return switch (entityType) {
|
||||
case ENTITY -> EntryType.ENTITY;
|
||||
case HINT -> EntryType.HINT;
|
||||
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
|
||||
case RECOMMENDATION -> EntryType.RECOMMENDATION;
|
||||
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
||||
|
||||
EntityLogEntry entityLogEntry;
|
||||
if (migratedEntity instanceof Image image) {
|
||||
entityLogEntry = createEntityLogEntry(image);
|
||||
} else if (migratedEntity instanceof TextEntity textEntity) {
|
||||
entityLogEntry = createEntityLogEntry(textEntity);
|
||||
} else if (migratedEntity instanceof PrecursorEntity entity) {
|
||||
entityLogEntry = createEntityLogEntry(entity);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
|
||||
}
|
||||
|
||||
entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
|
||||
entityLogEntry.setColor(redactionLogEntry.getColor());
|
||||
entityLogEntry.setChanges(redactionLogEntry.getChanges()
|
||||
.stream()
|
||||
.map(MigrationMapper::toEntityLogChanges)
|
||||
.toList());
|
||||
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
|
||||
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
|
||||
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
|
||||
if (redactionLogEntry.getLegalBasis() != null) {
|
||||
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
|
||||
}
|
||||
|
||||
if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
|
||||
entityLogEntry.setState(EntryState.IGNORED);
|
||||
}
|
||||
|
||||
return entityLogEntry;
|
||||
}
|
||||
|
||||
|
||||
private static boolean lastManualChangeIsRemoveLocally(EntityLogEntry entityLogEntry) {
|
||||
|
||||
return entityLogEntry.getManualChanges()
|
||||
.stream()
|
||||
.reduce((a, b) -> b)
|
||||
.filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
|
||||
.isPresent();
|
||||
}
|
||||
|
||||
|
||||
private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
|
||||
|
||||
if (ids == null) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
return ids.stream()
|
||||
.map(oldToNewIdMapping::get)
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
public EntityLogEntry createEntityLogEntry(Image image) {
|
||||
|
||||
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
|
||||
return EntityLogEntry.builder()
|
||||
.id(image.getId())
|
||||
.value(image.value())
|
||||
.type(image.type())
|
||||
.reason(image.buildReasonWithManualChangeDescriptions())
|
||||
.legalBasis(image.legalBasis())
|
||||
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
||||
.dictionaryEntry(false)
|
||||
.positions(positions)
|
||||
.containingNodeId(image.getTreeId())
|
||||
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
||||
.section(image.getManualOverwrite().getSection()
|
||||
.orElse(redactionLogEntry.getSection()))
|
||||
.textAfter(redactionLogEntry.getTextAfter())
|
||||
.textBefore(redactionLogEntry.getTextBefore())
|
||||
.imageHasTransparency(image.isTransparent())
|
||||
.state(buildEntryState(image))
|
||||
.entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||
.build();
|
||||
|
||||
}
|
||||
|
||||
|
||||
public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
|
||||
|
||||
return EntityLogEntry.builder()
|
||||
.id(precursorEntity.getId())
|
||||
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
|
||||
.legalBasis(precursorEntity.legalBasis())
|
||||
.value(precursorEntity.value())
|
||||
.type(precursorEntity.type())
|
||||
.state(buildEntryState(precursorEntity))
|
||||
.entryType(buildEntryType(precursorEntity))
|
||||
.section(precursorEntity.getManualOverwrite().getSection()
|
||||
.orElse(redactionLogEntry.getSection()))
|
||||
.textAfter(redactionLogEntry.getTextAfter())
|
||||
.textBefore(redactionLogEntry.getTextBefore())
|
||||
.containingNodeId(Collections.emptyList())
|
||||
.closestHeadline("")
|
||||
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
|
||||
.dictionaryEntry(precursorEntity.isDictionaryEntry())
|
||||
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
|
||||
.startOffset(-1)
|
||||
.endOffset(-1)
|
||||
.positions(precursorEntity.getManualOverwrite().getPositions()
|
||||
.orElse(precursorEntity.getEntityPosition())
|
||||
.stream()
|
||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||
.toList())
|
||||
.engines(Collections.emptySet())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public EntityLogEntry createEntityLogEntry(TextEntity entity) {
|
||||
|
||||
assert entity.getPositionsOnPagePerPage().size() == 1;
|
||||
List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
|
||||
return EntityLogEntry.builder()
|
||||
.id(entity.getId())
|
||||
.positions(rectanglesPerLine)
|
||||
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||
.legalBasis(entity.legalBasis())
|
||||
.value(entity.getManualOverwrite().getValue()
|
||||
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||
.type(entity.type())
|
||||
.section(entity.getManualOverwrite().getSection()
|
||||
.orElse(redactionLogEntry.getSection()))
|
||||
.textAfter(entity.getTextAfter())
|
||||
.textBefore(entity.getTextBefore())
|
||||
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
||||
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
||||
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
||||
.dictionaryEntry(entity.isDictionaryEntry())
|
||||
.startOffset(entity.getTextRange().start())
|
||||
.endOffset(entity.getTextRange().end())
|
||||
.dossierDictionaryEntry(entity.isDossierDictionaryEntry())
|
||||
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
|
||||
.state(buildEntryState(entity))
|
||||
.entryType(buildEntryType(entity))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private static List<Position> getRectanglesPerLine(TextEntity entity) {
|
||||
|
||||
return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
|
||||
.get(0).getRectanglePerLine()
|
||||
.stream()
|
||||
.map(rectangle2D -> new Position(rectangle2D,
|
||||
entity.getPositionsOnPagePerPage()
|
||||
.get(0).getPage().getNumber()))
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
|
||||
|
||||
return entity.getManualOverwrite().getPositions()
|
||||
.map(rects -> rects.stream()
|
||||
.map(r -> new Position(r.rectangle2D(), r.pageNumber()))
|
||||
.toList());
|
||||
}
|
||||
|
||||
|
||||
public boolean hasManualChangesOrComments() {
|
||||
|
||||
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
|
||||
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|
||||
|| hasManualChanges();
|
||||
}
|
||||
|
||||
|
||||
public boolean hasManualChanges() {
|
||||
|
||||
return !manualChanges.isEmpty();
|
||||
}
|
||||
|
||||
|
||||
public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
|
||||
|
||||
manualChanges.addAll(manualChangesToApply);
|
||||
manualChangesToApply.forEach(manualChange -> {
|
||||
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
|
||||
// Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
|
||||
// Instead, we add an add_locally change to the db.
|
||||
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
|
||||
.positions(manualResizeRedaction.getPositions())
|
||||
.annotationId(getNewId())
|
||||
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
|
||||
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
|
||||
.textAfter(manualResizeRedaction.getTextAfter())
|
||||
.textBefore(manualResizeRedaction.getTextBefore())
|
||||
.build();
|
||||
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
|
||||
} else {
|
||||
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
public ManualRedactionEntry buildManualRedactionEntry() {
|
||||
|
||||
assert hasManualChanges();
|
||||
|
||||
// currently we need to insert a manual redaction entry, whenever an entity has been resized.
|
||||
String user = manualChanges.stream()
|
||||
.filter(mc -> mc instanceof ManualResizeRedaction)
|
||||
.findFirst()
|
||||
.orElse(manualChanges.get(0)).getUser();
|
||||
|
||||
return ManualRedactionEntry.builder()
|
||||
.annotationId(newId)
|
||||
.fileId(fileId)
|
||||
.type(redactionLogEntry.getType())
|
||||
.value(redactionLogEntry.getValue())
|
||||
.reason(redactionLogEntry.getReason())
|
||||
.legalBasis(redactionLogEntry.getLegalBasis())
|
||||
.section(redactionLogEntry.getSection())
|
||||
.addToDictionary(false)
|
||||
.addToDossierDictionary(false)
|
||||
.rectangle(false)
|
||||
.positions(buildPositions(migratedEntity))
|
||||
.user(user)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private List<Rectangle> buildPositions(IEntity entity) {
|
||||
|
||||
if (entity instanceof TextEntity textEntity) {
|
||||
|
||||
var positionsOnPage = textEntity.getPositionsOnPagePerPage()
|
||||
.get(0);
|
||||
return positionsOnPage.getRectanglePerLine()
|
||||
.stream()
|
||||
.map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
|
||||
.toList();
|
||||
}
|
||||
if (entity instanceof PrecursorEntity pEntity) {
|
||||
|
||||
return pEntity.getManualOverwrite().getPositions()
|
||||
.orElse(pEntity.getEntityPosition())
|
||||
.stream()
|
||||
.map(p -> new Rectangle((float) p.rectangle2D().getX(),
|
||||
(float) p.rectangle2D().getY(),
|
||||
(float) p.rectangle2D().getWidth(),
|
||||
(float) p.rectangle2D().getHeight(),
|
||||
p.pageNumber()))
|
||||
.toList();
|
||||
}
|
||||
if (entity instanceof Image image) {
|
||||
|
||||
Rectangle2D position = image.getManualOverwrite().getPositions()
|
||||
.map(p -> p.get(0).rectangle2D())
|
||||
.orElse(image.getPosition());
|
||||
|
||||
return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
|
||||
|
||||
} else {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean needsManualEntry() {
|
||||
|
||||
return manualChanges.stream()
|
||||
.anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
|
||||
}
|
||||
|
||||
}
|
||||
-195
@@ -1,195 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
|
||||
public class PrecursorEntity implements IEntity {
|
||||
|
||||
// The id must be mapped into a TextEntity as is for comments to work correctly
|
||||
String id;
|
||||
String value;
|
||||
List<RectangleWithPage> entityPosition;
|
||||
String ruleIdentifier;
|
||||
String reason;
|
||||
String legalBasis;
|
||||
String type;
|
||||
String section;
|
||||
EntityType entityType;
|
||||
boolean applied;
|
||||
boolean isDictionaryEntry;
|
||||
boolean isDossierDictionaryEntry;
|
||||
boolean rectangle;
|
||||
Set<Engine> engines;
|
||||
|
||||
@Builder.Default
|
||||
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
|
||||
ManualChangeOverwrite manualOverwrite;
|
||||
|
||||
|
||||
public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromAnnotationRectangle)
|
||||
.toList();
|
||||
|
||||
var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
|
||||
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
|
||||
manualChangeOverwrite.addChange(manualRedactionEntry);
|
||||
return PrecursorEntity.builder()
|
||||
.id(manualRedactionEntry.getAnnotationId())
|
||||
.value(manualRedactionEntry.getValue())
|
||||
.entityPosition(rectangleWithPages)
|
||||
.ruleIdentifier("MAN.0.0")
|
||||
.reason(manualRedactionEntry.getReason())
|
||||
.legalBasis(manualRedactionEntry.getLegalBasis())
|
||||
.type(manualRedactionEntry.getType())
|
||||
.section(manualRedactionEntry.getSection())
|
||||
.entityType(entityType)
|
||||
.applied(true)
|
||||
.isDictionaryEntry(false)
|
||||
.isDossierDictionaryEntry(false)
|
||||
.rectangle(manualRedactionEntry.isRectangle())
|
||||
.manualOverwrite(manualChangeOverwrite)
|
||||
.engines(Set.of(Engine.MANUAL))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromEntityLogPosition)
|
||||
.toList();
|
||||
EntityType entityType = getEntityType(entityLogEntry.getEntryType());
|
||||
return PrecursorEntity.builder()
|
||||
.id(entityLogEntry.getId())
|
||||
.value(entityLogEntry.getValue())
|
||||
.entityPosition(rectangleWithPages)
|
||||
.ruleIdentifier(entityLogEntry.getMatchedRule())
|
||||
.reason(entityLogEntry.getReason())
|
||||
.legalBasis(entityLogEntry.getLegalBasis())
|
||||
.type(entityLogEntry.getType())
|
||||
.section(entityLogEntry.getSection())
|
||||
.entityType(entityType)
|
||||
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
|
||||
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
|
||||
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||
.engines(entityLogEntry.getEngines())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromEntityLogPosition)
|
||||
.toList();
|
||||
EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType())
|
||||
.orElse(EntryType.ENTITY);
|
||||
EntityType entityType = getEntityType(entryType);
|
||||
String value = Optional.ofNullable(importedRedaction.getValue())
|
||||
.orElse("");
|
||||
return PrecursorEntity.builder()
|
||||
.id(importedRedaction.getId())
|
||||
.value(value)
|
||||
.entityPosition(rectangleWithPages)
|
||||
.reason(Optional.ofNullable(importedRedaction.getReason())
|
||||
.orElse(""))
|
||||
.legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
|
||||
.orElse(""))
|
||||
.type(Optional.ofNullable(importedRedaction.getType())
|
||||
.orElse(IMPORTED_REDACTION_TYPE))
|
||||
.entityType(entityType)
|
||||
.isDictionaryEntry(false)
|
||||
.isDossierDictionaryEntry(false)
|
||||
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
|
||||
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||
.engines(Set.of(Engine.IMPORTED))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromAnnotationRectangle)
|
||||
.toList();
|
||||
return PrecursorEntity.builder()
|
||||
.id(UUID.randomUUID().toString())
|
||||
.value(manualResizeRedaction.getValue())
|
||||
.entityPosition(rectangleWithPages)
|
||||
.entityType(EntityType.ENTITY)
|
||||
.manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextRange getTextRange() {
|
||||
|
||||
return new TextRange(-1, -1);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
||||
return getManualOverwrite().getType()
|
||||
.orElse(type);
|
||||
}
|
||||
|
||||
|
||||
private static EntityType getEntityType(EntryType entryType) {
|
||||
|
||||
switch (entryType) {
|
||||
case FALSE_RECOMMENDATION -> {
|
||||
return EntityType.FALSE_RECOMMENDATION;
|
||||
}
|
||||
case FALSE_POSITIVE -> {
|
||||
return EntityType.FALSE_POSITIVE;
|
||||
}
|
||||
case HINT -> {
|
||||
return EntityType.HINT;
|
||||
}
|
||||
case RECOMMENDATION -> {
|
||||
return EntityType.RECOMMENDATION;
|
||||
}
|
||||
default -> {
|
||||
return EntityType.ENTITY;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
-45
@@ -1,45 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
|
||||
public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||
|
||||
public static RectangleWithPage fromRedactionLogRectangle(Rectangle rectangle) {
|
||||
|
||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||
}
|
||||
|
||||
|
||||
public static RectangleWithPage fromEntityLogPosition(Position position) {
|
||||
|
||||
return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position));
|
||||
}
|
||||
|
||||
|
||||
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(Rectangle rectangle) {
|
||||
|
||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(Position position) {
|
||||
|
||||
return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h());
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
|
||||
}
|
||||
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.component;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class Component {
|
||||
|
||||
RuleIdentifier matchedRule;
|
||||
String name;
|
||||
String value;
|
||||
String valueDescription;
|
||||
|
||||
List<Entity> references;
|
||||
|
||||
|
||||
public boolean addReference(Entity entity) {
|
||||
|
||||
return references.add(entity);
|
||||
}
|
||||
|
||||
|
||||
public boolean addReferences(Collection<Entity> entities) {
|
||||
|
||||
return references.addAll(entities);
|
||||
}
|
||||
|
||||
}
|
||||
-104
@@ -1,104 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.component;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class Entity {
|
||||
|
||||
String id;
|
||||
String type;
|
||||
EntryType entryType;
|
||||
EntryState state;
|
||||
String value;
|
||||
String reason;
|
||||
String matchedRule;
|
||||
String legalBasis;
|
||||
|
||||
boolean imported;
|
||||
|
||||
SemanticNode containingNode;
|
||||
String section;
|
||||
float[] color;
|
||||
|
||||
List<Position> positions;
|
||||
|
||||
String textBefore;
|
||||
String textAfter;
|
||||
|
||||
int startOffset;
|
||||
int endOffset;
|
||||
int length;
|
||||
|
||||
boolean imageHasTransparency;
|
||||
|
||||
boolean isDictionaryEntry;
|
||||
boolean isDossierDictionaryEntry;
|
||||
|
||||
boolean excluded;
|
||||
|
||||
List<Change> changes;
|
||||
|
||||
List<ManualChange> manualChanges;
|
||||
|
||||
Set<Engine> engines;
|
||||
|
||||
Set<String> reference;
|
||||
|
||||
Set<String> importedRedactionIntersections;
|
||||
|
||||
|
||||
public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
|
||||
|
||||
return Entity.builder()
|
||||
.id(e.getId())
|
||||
.type(e.getType())
|
||||
.entryType(e.getEntryType())
|
||||
.state(e.getState())
|
||||
.value(e.getValue())
|
||||
.reason(e.getReason())
|
||||
.matchedRule(e.getMatchedRule())
|
||||
.legalBasis(e.getLegalBasis())
|
||||
.imported(e.isImported())
|
||||
.section(e.getSection())
|
||||
.color(e.getColor())
|
||||
.positions(e.getPositions())
|
||||
.containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
|
||||
.textBefore(e.getTextBefore())
|
||||
.textAfter(e.getTextAfter())
|
||||
.startOffset(e.getStartOffset())
|
||||
.endOffset(e.getEndOffset())
|
||||
.length(Optional.ofNullable(e.getValue()).orElse("").length())
|
||||
.imageHasTransparency(e.isImageHasTransparency())
|
||||
.isDictionaryEntry(e.isDictionaryEntry())
|
||||
.isDossierDictionaryEntry(e.isDossierDictionaryEntry())
|
||||
.excluded(e.isExcluded())
|
||||
.changes(e.getChanges())
|
||||
.manualChanges(e.getManualChanges())
|
||||
.engines(e.getEngines())
|
||||
.reference(e.getReference())
|
||||
.importedRedactionIntersections(e.getImportedRedactionIntersections())
|
||||
.build();
|
||||
}
|
||||
|
||||
}
|
||||
-251
@@ -1,251 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public class ManualChangeOverwrite {
|
||||
|
||||
private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
|
||||
ManualRedactionEntry.class, "created by manual change", //
|
||||
ManualLegalBasisChange.class, "legal basis was manually changed", //
|
||||
ManualResizeRedaction.class, "resized by manual override", //
|
||||
ManualForceRedaction.class, "forced by manual override", //
|
||||
IdRemoval.class, "removed by manual override", //
|
||||
ManualRecategorization.class, "recategorized by manual override");
|
||||
|
||||
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
||||
boolean changed;
|
||||
List<String> descriptions;
|
||||
String type;
|
||||
String legalBasis;
|
||||
String section;
|
||||
String value;
|
||||
List<RectangleWithPage> positions;
|
||||
Boolean applied;
|
||||
Boolean removed;
|
||||
Boolean ignored;
|
||||
Boolean resized;
|
||||
Boolean recategorized;
|
||||
EntityType entityType;
|
||||
|
||||
|
||||
public ManualChangeOverwrite() {
|
||||
|
||||
this(EntityType.ENTITY);
|
||||
}
|
||||
|
||||
|
||||
public ManualChangeOverwrite(EntityType entityType) {
|
||||
|
||||
this.entityType = entityType;
|
||||
}
|
||||
|
||||
|
||||
public void calculateCurrentOverride() {
|
||||
|
||||
if (!changed) {
|
||||
return;
|
||||
}
|
||||
List<BaseAnnotation> sortedManualChanges = getManualChangeLog();
|
||||
updateFields(sortedManualChanges);
|
||||
}
|
||||
|
||||
|
||||
public List<BaseAnnotation> getManualChangeLog() {
|
||||
|
||||
if (!changed) {
|
||||
return manualChanges;
|
||||
}
|
||||
manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
|
||||
updateFields(manualChanges);
|
||||
// make list unmodifiable.
|
||||
return manualChanges.stream()
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
private void updateFields(List<BaseAnnotation> sortedManualChanges) {
|
||||
|
||||
descriptions = new LinkedList<>();
|
||||
|
||||
for (BaseAnnotation manualChange : sortedManualChanges) {
|
||||
// ManualRedactionEntries are created prior to rule execution in analysis service.
|
||||
|
||||
if (manualChange instanceof IdRemoval) {
|
||||
applied = false;
|
||||
if (sortedManualChanges.get(0) instanceof ManualRedactionEntry) { // set to removed if it's a Manual Redaction
|
||||
removed = true;
|
||||
} else {
|
||||
ignored = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) {
|
||||
setToApplied();
|
||||
legalBasis = manualRedactionEntry.getLegalBasis();
|
||||
}
|
||||
|
||||
if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
|
||||
setToApplied();
|
||||
legalBasis = manualForceRedaction.getLegalBasis();
|
||||
}
|
||||
|
||||
if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
|
||||
section = manualLegalBasisChange.getSection();
|
||||
legalBasis = manualLegalBasisChange.getLegalBasis();
|
||||
value = manualLegalBasisChange.getValue();
|
||||
}
|
||||
|
||||
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
|
||||
// resizing logic for TextEntities and Images happens in ManualChangesApplicationService.
|
||||
resized = true;
|
||||
// This is only for not found Manual Entities.
|
||||
value = manualResizeRedaction.getValue();
|
||||
positions = manualResizeRedaction.getPositions()
|
||||
.stream()
|
||||
.map(RectangleWithPage::fromAnnotationRectangle)
|
||||
.toList();
|
||||
}
|
||||
|
||||
if (manualChange instanceof ManualRecategorization recategorization) {
|
||||
recategorized = true;
|
||||
type = recategorization.getType();
|
||||
if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
|
||||
legalBasis = recategorization.getLegalBasis();
|
||||
}
|
||||
}
|
||||
|
||||
descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
|
||||
}
|
||||
changed = false;
|
||||
}
|
||||
|
||||
|
||||
private void setToApplied() {
|
||||
|
||||
removed = false;
|
||||
ignored = false;
|
||||
if (!isHint(entityType)) {
|
||||
applied = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void addChange(BaseAnnotation manualChange) {
|
||||
|
||||
changed = true;
|
||||
manualChanges.add(manualChange);
|
||||
}
|
||||
|
||||
|
||||
public void addChanges(List<BaseAnnotation> manualChangeLog) {
|
||||
|
||||
changed = true;
|
||||
manualChanges.addAll(manualChangeLog);
|
||||
}
|
||||
|
||||
|
||||
public Optional<String> getLegalBasis() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return legalBasis == null ? Optional.empty() : Optional.of(legalBasis);
|
||||
}
|
||||
|
||||
|
||||
public Optional<String> getType() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return type == null ? Optional.empty() : Optional.of(type);
|
||||
}
|
||||
|
||||
|
||||
public Optional<String> getSection() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return section == null ? Optional.empty() : Optional.of(section);
|
||||
}
|
||||
|
||||
|
||||
public Optional<String> getValue() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return value == null ? Optional.empty() : Optional.of(value);
|
||||
}
|
||||
|
||||
|
||||
public Optional<Boolean> getApplied() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return applied == null ? Optional.empty() : Optional.of(applied);
|
||||
}
|
||||
|
||||
|
||||
public Optional<Boolean> getRemoved() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return removed == null ? Optional.empty() : Optional.of(removed);
|
||||
}
|
||||
|
||||
|
||||
public Optional<Boolean> getIgnored() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return ignored == null ? Optional.empty() : Optional.of(ignored);
|
||||
}
|
||||
|
||||
|
||||
public Optional<Boolean> getResized() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return resized == null ? Optional.empty() : Optional.of(resized);
|
||||
}
|
||||
|
||||
|
||||
public Optional<Boolean> getRecategorized() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return recategorized == null ? Optional.empty() : Optional.of(recategorized);
|
||||
}
|
||||
|
||||
|
||||
public List<String> getDescriptions() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return descriptions == null ? Collections.emptyList() : descriptions;
|
||||
}
|
||||
|
||||
|
||||
public Optional<List<RectangleWithPage>> getPositions() {
|
||||
|
||||
calculateCurrentOverride();
|
||||
return positions == null ? Optional.empty() : Optional.of(positions);
|
||||
}
|
||||
|
||||
|
||||
private boolean isHint(EntityType entityType) {
|
||||
|
||||
return entityType.equals(EntityType.HINT);
|
||||
}
|
||||
|
||||
}
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
|
||||
import com.iqser.red.service.redaction.v1.server.model.drools.RuleType;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public final class MatchedRule implements Comparable<MatchedRule> {
|
||||
|
||||
public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
|
||||
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
|
||||
public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
|
||||
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
|
||||
|
||||
RuleIdentifier ruleIdentifier;
|
||||
@Builder.Default
|
||||
String reason = "";
|
||||
@Builder.Default
|
||||
String legalBasis = "";
|
||||
boolean writeValueWithLineBreaks;
|
||||
boolean applied;
|
||||
boolean removed;
|
||||
boolean ignored;
|
||||
|
||||
@Builder.Default
|
||||
Set<TextEntity> references = Collections.emptySet();
|
||||
|
||||
|
||||
public static MatchedRule empty() {
|
||||
|
||||
return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
|
||||
}
|
||||
|
||||
|
||||
public MatchedRule asSkippedIfApplied() {
|
||||
|
||||
if (!this.isApplied()) {
|
||||
return this;
|
||||
}
|
||||
return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
|
||||
.writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
|
||||
.legalBasis(this.getLegalBasis())
|
||||
.reason(this.getReason())
|
||||
.references(this.getReferences())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int compareTo(MatchedRule matchedRule) {
|
||||
|
||||
// Only the highest ranked rule is actually applied, this method defines the highest order.
|
||||
// First, it compares the Rule Type, RULE_TYPE_PRIORITIES defines the order of types.
|
||||
// Types not in the list have the lowest priority.
|
||||
// The ones in the list are technical exceptions and should override any other Rule.
|
||||
// Aside from them Entities should never match from more than one type!
|
||||
// E.g. a CBI_author entity should **always** only match CBI.*.* rules.
|
||||
// Otherwise, something went wrong with the rules. :)
|
||||
RuleIdentifier otherRuleIdentifier = matchedRule.getRuleIdentifier();
|
||||
boolean thisInList = RULE_TYPE_PRIORITIES.contains(this.getRuleIdentifier().type());
|
||||
boolean otherInList = RULE_TYPE_PRIORITIES.contains(otherRuleIdentifier.type());
|
||||
|
||||
// Compare the types
|
||||
if (thisInList && !otherInList) {
|
||||
return -1;
|
||||
} else if (!thisInList && otherInList) {
|
||||
return 1;
|
||||
} else if (thisInList && otherInList) {
|
||||
int thisIndex = RULE_TYPE_PRIORITIES.indexOf(this.getRuleIdentifier().type());
|
||||
int otherIndex = RULE_TYPE_PRIORITIES.indexOf(otherRuleIdentifier.type());
|
||||
return Integer.compare(thisIndex, otherIndex);
|
||||
}
|
||||
// Then compare the unit
|
||||
if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) {
|
||||
return Integer.compare(otherRuleIdentifier.unit(), ruleIdentifier.unit());
|
||||
}
|
||||
// Then compare the id inside the unit
|
||||
return Integer.compare(otherRuleIdentifier.id(), ruleIdentifier.id());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return "MatchedRule[ruleIdentifier=" + ruleIdentifier + ", reason=" + reason + ", legalBasis=" + legalBasis + ", applied=" + applied + ", writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", references=" + references + ']';
|
||||
}
|
||||
|
||||
}
|
||||
-237
@@ -1,237 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.entity;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
|
||||
public class TextEntity implements IEntity {
|
||||
|
||||
// primary key
|
||||
@EqualsAndHashCode.Include
|
||||
final String id;
|
||||
// primary key end
|
||||
|
||||
TextRange textRange;
|
||||
@Builder.Default
|
||||
List<TextRange> duplicateTextRanges = new ArrayList<>();
|
||||
String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
|
||||
final EntityType entityType;
|
||||
|
||||
@Builder.Default
|
||||
final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
|
||||
final ManualChangeOverwrite manualOverwrite;
|
||||
|
||||
boolean dictionaryEntry;
|
||||
boolean dossierDictionaryEntry;
|
||||
|
||||
@Builder.Default
|
||||
Set<Engine> engines = new HashSet<>();
|
||||
|
||||
// inferred on graph insertion
|
||||
String value;
|
||||
String textBefore;
|
||||
String textAfter;
|
||||
@Builder.Default
|
||||
Set<Page> pages = new HashSet<>();
|
||||
List<PositionOnPage> positionsOnPagePerPage;
|
||||
@Builder.Default
|
||||
List<SemanticNode> intersectingNodes = new LinkedList<>();
|
||||
SemanticNode deepestFullyContainingNode;
|
||||
|
||||
|
||||
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
|
||||
}
|
||||
|
||||
|
||||
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
|
||||
|
||||
return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
|
||||
}
|
||||
|
||||
|
||||
private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
|
||||
|
||||
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
|
||||
return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
|
||||
}
|
||||
|
||||
|
||||
public void addTextRange(TextRange textRange) {
|
||||
|
||||
duplicateTextRanges.add(textRange);
|
||||
}
|
||||
|
||||
public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
|
||||
|
||||
return intersectingNodes.stream().anyMatch(clazz::isInstance);
|
||||
}
|
||||
|
||||
|
||||
public boolean occursInNode(SemanticNode semanticNode) {
|
||||
|
||||
return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode));
|
||||
}
|
||||
|
||||
|
||||
public boolean isType(String type) {
|
||||
|
||||
return type().equals(type);
|
||||
}
|
||||
|
||||
|
||||
public boolean isAnyType(List<String> types) {
|
||||
|
||||
return types.contains(type());
|
||||
}
|
||||
|
||||
|
||||
public void addIntersectingNode(SemanticNode containingNode) {
|
||||
|
||||
intersectingNodes.add(containingNode);
|
||||
}
|
||||
|
||||
|
||||
public String getValueWithLineBreaks() {
|
||||
|
||||
return getDeepestFullyContainingNode().getTextBlock().subSequenceWithLineBreaks(getTextRange());
|
||||
}
|
||||
|
||||
|
||||
public void removeFromGraph() {
|
||||
|
||||
intersectingNodes.forEach(node -> node.getEntities().remove(this));
|
||||
pages.forEach(page -> page.getEntities().remove(this));
|
||||
intersectingNodes = new LinkedList<>();
|
||||
deepestFullyContainingNode = null;
|
||||
pages = new HashSet<>();
|
||||
remove("FINAL.0.0", "removed completely");
|
||||
}
|
||||
|
||||
|
||||
public List<PositionOnPage> getPositionsOnPagePerPage() {
|
||||
|
||||
if (positionsOnPagePerPage == null || positionsOnPagePerPage.isEmpty()) {
|
||||
Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(textRange);
|
||||
|
||||
Page firstPage = rectanglesPerLinePerPage.keySet()
|
||||
.stream()
|
||||
.min(Comparator.comparingInt(Page::getNumber))
|
||||
.orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
|
||||
|
||||
positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
|
||||
}
|
||||
return positionsOnPagePerPage;
|
||||
}
|
||||
|
||||
|
||||
private static PositionOnPage buildPositionOnPage(Page firstPage, String id, Map.Entry<Page, List<Rectangle2D>> entry) {
|
||||
|
||||
if (entry.getKey().equals(firstPage)) {
|
||||
return new PositionOnPage(id, entry.getKey(), entry.getValue());
|
||||
} else {
|
||||
return new PositionOnPage(id + "-" + entry.getKey().getNumber(), entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean containedBy(TextEntity textEntity) {
|
||||
|
||||
return this.textRange.containedBy(textEntity.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
public boolean contains(TextEntity textEntity) {
|
||||
|
||||
return this.textRange.contains(textEntity.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(TextEntity textEntity) {
|
||||
|
||||
return this.textRange.intersects(textEntity.getTextRange());
|
||||
}
|
||||
|
||||
|
||||
public void addEngine(Engine engine) {
|
||||
|
||||
engines.add(engine);
|
||||
}
|
||||
|
||||
|
||||
public void addEngines(Set<Engine> engines) {
|
||||
|
||||
this.engines.addAll(engines);
|
||||
}
|
||||
|
||||
|
||||
public boolean matchesAnnotationId(String manualRedactionId) {
|
||||
|
||||
return getPositionsOnPagePerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("Entity[\"");
|
||||
sb.append(value);
|
||||
sb.append("\", ");
|
||||
sb.append(textRange);
|
||||
sb.append(", pages[");
|
||||
pages.forEach(page -> {
|
||||
sb.append(page.getNumber());
|
||||
sb.append(", ");
|
||||
});
|
||||
sb.delete(sb.length() - 2, sb.length());
|
||||
sb.append("], type = \"");
|
||||
sb.append(type());
|
||||
sb.append("\", EntityType.");
|
||||
sb.append(entityType);
|
||||
sb.append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
||||
return getManualOverwrite().getType().orElse(type);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String value() {
|
||||
|
||||
return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
|
||||
}
|
||||
|
||||
}
|
||||
-165
@@ -1,165 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.PriorityQueue;
|
||||
import java.util.Set;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
||||
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Image implements GenericSemanticNode, IEntity {
|
||||
|
||||
@Builder.Default
|
||||
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
List<Integer> treeId;
|
||||
String id;
|
||||
|
||||
ImageType imageType;
|
||||
boolean transparent;
|
||||
Rectangle2D position;
|
||||
|
||||
@Builder.Default
|
||||
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
|
||||
|
||||
@Builder.Default
|
||||
ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
Page page;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
DocumentTree documentTree;
|
||||
|
||||
@Builder.Default
|
||||
@EqualsAndHashCode.Exclude
|
||||
Set<TextEntity> entities = new HashSet<>();
|
||||
|
||||
|
||||
@Override
|
||||
public NodeType getType() {
|
||||
|
||||
return NodeType.IMAGE;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextBlock getTextBlock() {
|
||||
|
||||
return streamAllSubNodes().filter(SemanticNode::isLeaf)
|
||||
.map(SemanticNode::getLeafTextBlock)
|
||||
.collect(new TextBlockCollector());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Set<Page> getPages() {
|
||||
|
||||
return Collections.singleton(page);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public TextRange getTextRange() {
|
||||
|
||||
return GenericSemanticNode.super.getTextRange();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String type() {
|
||||
|
||||
return getManualOverwrite().getType()
|
||||
.orElse(imageType.toString());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<Page, Rectangle2D> getBBox() {
|
||||
|
||||
Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
|
||||
bBoxPerPage.put(page, position);
|
||||
return bBoxPerPage;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String getValue() {
|
||||
|
||||
return NodeType.IMAGE + ":" + camelCase(imageType.toString());
|
||||
}
|
||||
|
||||
|
||||
private String camelCase(String name) {
|
||||
|
||||
return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH);
|
||||
}
|
||||
|
||||
|
||||
public boolean mostlyContainedBy(Image image, double containmentThreshold) {
|
||||
|
||||
Map<Page, Rectangle2D> bboxImage = image.getBBox();
|
||||
Map<Page, Rectangle2D> bbox = this.getBBox();
|
||||
//image needs to be on the same page
|
||||
if(bboxImage.get(this.page) != null) {
|
||||
Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
|
||||
double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
|
||||
double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
|
||||
return (calculatedIntersection / area) > containmentThreshold;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public boolean mostlyContains(Image image, double containmentThreshold) {
|
||||
|
||||
Map<Page, Rectangle2D> bboxImage = image.getBBox();
|
||||
Map<Page, Rectangle2D> bbox = this.getBBox();
|
||||
Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
|
||||
double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
|
||||
double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
|
||||
return (area / calculatedIntersection) > containmentThreshold;
|
||||
}
|
||||
|
||||
|
||||
public int length() {
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.Data;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class BasicQuery {
|
||||
|
||||
String name;
|
||||
int line;
|
||||
String code;
|
||||
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import org.drools.drl.ast.descr.AttributeDescr;
|
||||
import org.drools.drl.ast.descr.RuleDescr;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.Getter;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||
public class BasicRule {
|
||||
|
||||
@EqualsAndHashCode.Include
|
||||
RuleIdentifier identifier;
|
||||
String name;
|
||||
String code;
|
||||
String agendaGroup;
|
||||
int line;
|
||||
|
||||
|
||||
public static BasicRule fromRuleDescr(RuleDescr rule, String rulesString) {
|
||||
|
||||
RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
|
||||
String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
|
||||
String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
|
||||
String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
|
||||
return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return "BasicRule[identifier=" + identifier + ", name=" + name + ", code=" + code + ']';
|
||||
}
|
||||
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
||||
public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
|
||||
|
||||
public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
|
||||
|
||||
return ruleUnits.stream()
|
||||
.filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
|
||||
}
|
||||
|
||||
}
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@FieldDefaults(level = AccessLevel.PRIVATE)
|
||||
public final class RuleFileBluePrint {
|
||||
|
||||
String imports;
|
||||
int importLine;
|
||||
String globals;
|
||||
int globalsLine;
|
||||
List<BasicQuery> queries;
|
||||
List<RuleClass> ruleClasses;
|
||||
DroolsSyntaxValidation droolsSyntaxValidation;
|
||||
|
||||
|
||||
public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
|
||||
|
||||
return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
|
||||
}
|
||||
|
||||
|
||||
public Set<String> getImportSplitByKeyword() {
|
||||
|
||||
return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
|
||||
|
||||
if (Objects.isNull(ruleIdentifier.unit())) {
|
||||
return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits)
|
||||
.orElse(Collections.emptyList())
|
||||
.stream()
|
||||
.flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
|
||||
.toList();
|
||||
}
|
||||
return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()))
|
||||
.map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
|
||||
.orElse(Stream.empty())
|
||||
.toList();
|
||||
}
|
||||
|
||||
|
||||
public List<RuleIdentifier> getAllRuleIdentifiers() {
|
||||
|
||||
return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
|
||||
public Stream<BasicRule> streamAllRules() {
|
||||
|
||||
return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
return "RuleFileBluePrint[imports=" + imports + ", globals=" + globals + ", queries=" + queries + ", ruleClasses=" + ruleClasses + ']';
|
||||
}
|
||||
|
||||
}
|
||||
-81
@@ -1,81 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import lombok.NonNull;
|
||||
|
||||
public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
|
||||
|
||||
public static RuleIdentifier fromName(String name) {
|
||||
|
||||
String identifier = name.split(":")[0];
|
||||
return fromString(identifier);
|
||||
}
|
||||
|
||||
|
||||
public static RuleIdentifier fromString(String identifier) {
|
||||
|
||||
String[] values = identifier.split("\\.");
|
||||
if (values.length != 3) {
|
||||
throw new IllegalArgumentException(String.format("Identifier %s must conform to the pattern \"[a-zA-Z0-9]+.\\d+.\\d+\"", identifier));
|
||||
}
|
||||
RuleType type = RuleType.fromString(values[0]);
|
||||
Integer group = Integer.parseInt(values[1]);
|
||||
Integer id = Integer.parseInt(values[2]);
|
||||
return new RuleIdentifier(type, group, id);
|
||||
}
|
||||
|
||||
|
||||
public static RuleIdentifier empty() {
|
||||
|
||||
return new RuleIdentifier(new RuleType(""), null, null);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
|
||||
* Therefore, to compare RuleIdentifiers one should always use the function matches.
|
||||
*
|
||||
* @param ruleIdentifier RuleIdentifier to check if this one matches it.
|
||||
* @return true, if all fields match. If a field is null, it matches any field.
|
||||
*/
|
||||
public boolean matches(RuleIdentifier ruleIdentifier) {
|
||||
|
||||
return ruleIdentifier.type().equals(this.type()) && //
|
||||
(Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
|
||||
(Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
|
||||
if (type.name().isBlank() || type.name().isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(type().name());
|
||||
if (Objects.nonNull(unit())) {
|
||||
sb.append(".");
|
||||
sb.append(unit());
|
||||
} else {
|
||||
sb.append(".*");
|
||||
}
|
||||
if (Objects.nonNull(id())) {
|
||||
sb.append(".");
|
||||
sb.append(id());
|
||||
} else {
|
||||
sb.append(".*");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
public String toRuleUnitString() {
|
||||
|
||||
return String.format("Rule unit: %s.%d", type.name(), unit);
|
||||
}
|
||||
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public record RuleType(String name) {
|
||||
|
||||
static Pattern alphaNumeric = Pattern.compile("^[a-zA-Z0-9]+$");
|
||||
|
||||
|
||||
public static RuleType fromString(String value) {
|
||||
|
||||
if (!alphaNumeric.matcher(value).matches()) {
|
||||
throw new IllegalArgumentException(String.format("The rule type %s must only contain numbers and letters!", value));
|
||||
}
|
||||
return new RuleType(value);
|
||||
}
|
||||
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.drools;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record RuleUnit(Integer unit, List<BasicRule> rules) {
|
||||
|
||||
}
|
||||
+3
-1
@@ -1,11 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
+6
-32
@@ -1,11 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import org.springframework.amqp.core.Queue;
|
||||
import org.springframework.amqp.core.QueueBuilder;
|
||||
@@ -13,12 +8,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class MessagingConfiguration {
|
||||
|
||||
public static final String REDACTION_QUEUE = "redactionQueue";
|
||||
public static final String REDACTION_DQL = "redactionDQL";
|
||||
|
||||
public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
|
||||
|
||||
public static final String X_ERROR_INFO_HEADER = "x-error-message";
|
||||
public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
|
||||
|
||||
@@ -39,20 +37,6 @@ public class MessagingConfiguration {
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue migrationQueue() {
|
||||
|
||||
return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue migrationDLQ() {
|
||||
|
||||
return QueueBuilder.durable(MIGRATION_DLQ).build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionQueue() {
|
||||
|
||||
@@ -70,16 +54,6 @@ public class MessagingConfiguration {
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public Queue redactionAnalysisResponseQueue() {
|
||||
|
||||
return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
|
||||
.withArgument("x-dead-letter-exchange", "")
|
||||
.withArgument("x-dead-letter-routing-key", REDACTION_DQL)
|
||||
.maxPriority(2)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
public Queue redactionDeadLetterQueue() {
|
||||
|
||||
+3
-1
@@ -1,11 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
|
||||
|
||||
import org.springframework.amqp.core.Message;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
|
||||
import org.springframework.amqp.rabbit.annotation.RabbitListener;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
|
||||
+33
-45
@@ -1,8 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.server.queue;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
|
||||
import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
|
||||
import static java.lang.String.format;
|
||||
@@ -21,11 +21,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
@@ -39,8 +38,7 @@ public class RedactionMessageReceiver {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final AnalyzeService analyzeService;
|
||||
private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
|
||||
private final RulesClient rulesClient;
|
||||
private final UnprocessedChangesService unprocessedChangesService;
|
||||
private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
|
||||
|
||||
|
||||
@SneakyThrows
|
||||
@@ -69,16 +67,14 @@ public class RedactionMessageReceiver {
|
||||
@SneakyThrows
|
||||
public void receiveAnalyzeRequest(AnalyzeRequest analyzeRequest, boolean priority) {
|
||||
|
||||
log.info("");
|
||||
log.info("Processing AnalyzeRequest for file: {}, priority: {} ", analyzeRequest.getFileId(), priority);
|
||||
log.info("");
|
||||
AnalyzeResult result = new AnalyzeResult();
|
||||
boolean shouldRespond = true;
|
||||
log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
|
||||
AnalyzeResult result;
|
||||
|
||||
try {
|
||||
switch (analyzeRequest.getMessageType()) {
|
||||
|
||||
case ANALYSE:
|
||||
log.info("");
|
||||
log.info("------------------------------Full Analysis---------------------------------------");
|
||||
log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug(analyzeRequest.getManualRedactions().toString());
|
||||
@@ -91,6 +87,7 @@ public class RedactionMessageReceiver {
|
||||
break;
|
||||
|
||||
case REANALYSE:
|
||||
log.info("");
|
||||
log.info("------------------------------Reanalysis------------------------------------------");
|
||||
log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug(analyzeRequest.getManualRedactions().toString());
|
||||
@@ -101,50 +98,41 @@ public class RedactionMessageReceiver {
|
||||
format("%.2f", result.getDuration() / 1000.0));
|
||||
log.info("----------------------------------------------------------------------------------");
|
||||
break;
|
||||
case SURROUNDING_TEXT_ANALYSIS:
|
||||
log.info("------------------------------Surrounding Text Analysis------------------------------------------");
|
||||
log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
|
||||
log.debug(analyzeRequest.getManualRedactions().toString());
|
||||
unprocessedChangesService.analyseSurroundingText(analyzeRequest);
|
||||
log.info("Successful Surrounding Text Analysis dossier {} file {} ",
|
||||
|
||||
case SURROUNDING_TEXT:
|
||||
log.info("");
|
||||
log.info("------------------------------Add surrounding Text--------------------------------");
|
||||
log.info("Starting surrounding text search for entries {} ", analyzeRequest.getManualRedactions().getEntriesToAdd());
|
||||
result = manualRedactionSurroundingTextService.addSurroundingText(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
analyzeRequest.getManualRedactions());
|
||||
log.info("Successfully added surrounding text for manual redaction in dossierId {} and fileId {} took: {} s",
|
||||
analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId());
|
||||
log.info("-------------------------------------------------------------------------------------------------");
|
||||
shouldRespond = false;
|
||||
analyzeRequest.getFileId(),
|
||||
format("%.2f", result.getDuration() / 1000.0));
|
||||
log.info("----------------------------------------------------------------------------------");
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
|
||||
}
|
||||
log.info("");
|
||||
if (shouldRespond) {
|
||||
result.setMessageType(analyzeRequest.getMessageType());
|
||||
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
|
||||
}
|
||||
} catch (DroolsTimeoutException droolsTimeoutException) {
|
||||
if (!droolsTimeoutException.isReported()) {
|
||||
rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId(), droolsTimeoutException.getRuleFileType());
|
||||
}
|
||||
sendAnalysisFailed(analyzeRequest, priority, droolsTimeoutException);
|
||||
|
||||
result.setMessageType(analyzeRequest.getMessageType());
|
||||
fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
|
||||
|
||||
} catch (Exception e) {
|
||||
sendAnalysisFailed(analyzeRequest, priority, e);
|
||||
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
|
||||
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
|
||||
|
||||
log.warn("Failed to process analyze request: {}", analyzeRequest, e);
|
||||
var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
|
||||
analyzeRequest.getFileId(),
|
||||
new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
|
||||
}
|
||||
|
||||
|
||||
@RabbitHandler
|
||||
@RabbitListener(queues = REDACTION_DQL)
|
||||
public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
|
||||
public void receiveAnalyzeRequestDQL(Message in) throws IOException {
|
||||
|
||||
var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
|
||||
log.info("Failed to process analyze request: {}", analyzeRequest);
|
||||
|
||||
+298
@@ -0,0 +1,298 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.adapter;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static java.util.stream.Collectors.groupingBy;
|
||||
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
|
||||
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
|
||||
import com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class CustomEntityCreationAdapter {
|
||||
|
||||
private static final double MATCH_THRESHOLD = 5; // sum of distances for each corner
|
||||
private final EntityCreationService entityCreationService;
|
||||
|
||||
|
||||
@Autowired
|
||||
public CustomEntityCreationAdapter(EntityEnrichmentService entityEnrichmentService) {
|
||||
|
||||
entityCreationService = new EntityCreationService(entityEnrichmentService);
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
|
||||
|
||||
List<EntityIdentifier> entityIdentifiers = redactionLog.getRedactionLogEntry().stream().map(EntityIdentifier::fromRedactionLogEntry).toList();
|
||||
return toRedactionEntity(entityIdentifiers, node);
|
||||
}
|
||||
|
||||
|
||||
public List<RedactionEntity> createRedactionEntities(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) {
|
||||
|
||||
List<EntityIdentifier> entityIdentifiers = manualRedactionEntries.stream()
|
||||
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
|
||||
.map(EntityIdentifier::fromManualRedactionEntry)
|
||||
.toList();
|
||||
|
||||
return toRedactionEntity(entityIdentifiers, node);
|
||||
}
|
||||
|
||||
|
||||
private List<RedactionEntity> toRedactionEntity(List<EntityIdentifier> entityIdentifiers, SemanticNode node) {
|
||||
|
||||
Set<Integer> pageNumbers = entityIdentifiers.stream().flatMap(entry -> entry.entityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
|
||||
Set<String> entryValues = entityIdentifiers.stream().map(EntityIdentifier::value).map(String::toLowerCase).collect(Collectors.toSet());
|
||||
|
||||
Map<String, List<RedactionEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
|
||||
assert allValuesFound(tempEntitiesByValue, entryValues);
|
||||
|
||||
List<RedactionEntity> correctEntities = entityIdentifiers.stream()
|
||||
.map(entityIdentifier -> findClosestEntity(entityIdentifier, tempEntitiesByValue).map(tempEntity -> createCorrectEntity(entityIdentifier,
|
||||
node,
|
||||
tempEntity.getBoundary())))
|
||||
.filter(Optional::isPresent)
|
||||
.map(Optional::get)
|
||||
.toList();
|
||||
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(RedactionEntity::removeFromGraph);
|
||||
return correctEntities;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
|
||||
*
|
||||
* @param entityIdentifier The entity identifier for the RedactionEntity.
|
||||
* @param node The SemanticNode associated with the RedactionEntity.
|
||||
* @param closestBoundary The closest Boundary to the RedactionEntity.
|
||||
* @return The created correct RedactionEntity.
|
||||
*/
|
||||
private RedactionEntity createCorrectEntity(EntityIdentifier entityIdentifier, SemanticNode node, Boundary closestBoundary) {
|
||||
|
||||
RedactionEntity correctEntity = entityCreationService.forceByBoundary(closestBoundary, entityIdentifier.type(), entityIdentifier.entityType, node);
|
||||
|
||||
if (entityIdentifier.redacted()) {
|
||||
correctEntity.force(entityIdentifier.ruleIdentifier(), entityIdentifier.reason(), entityIdentifier.legalBasis());
|
||||
} else {
|
||||
correctEntity.skip(entityIdentifier.ruleIdentifier(), entityIdentifier.reason());
|
||||
}
|
||||
correctEntity.setDictionaryEntry(entityIdentifier.isDictionaryEntry());
|
||||
correctEntity.setDossierDictionaryEntry(entityIdentifier.isDossierDictionaryEntry());
|
||||
return correctEntity;
|
||||
}
|
||||
|
||||
|
||||
private Optional<RedactionEntity> findClosestEntity(EntityIdentifier identifier, Map<String, List<RedactionEntity>> entitiesWithSameValue) {
|
||||
|
||||
List<RedactionEntity> possibleEntities = entitiesWithSameValue.get(identifier.value().toLowerCase(Locale.ROOT));
|
||||
|
||||
if (possibleEntities == null || possibleEntities.isEmpty()) {
|
||||
log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.value());
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
Optional<RedactionEntity> optionalClosestEntity = possibleEntities.stream()
|
||||
.filter(entity -> pagesMatch(entity, identifier.entityPosition()))
|
||||
.min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.entityPosition(), entity)));
|
||||
|
||||
if (optionalClosestEntity.isEmpty()) {
|
||||
log.warn("No Entity with value {} found on page {}", identifier.value(), identifier.entityPosition());
|
||||
return optionalClosestEntity;
|
||||
}
|
||||
|
||||
RedactionEntity closestEntity = optionalClosestEntity.get();
|
||||
double distance = calculateMinDistance(identifier.entityPosition(), closestEntity);
|
||||
if (distance > MATCH_THRESHOLD) {
|
||||
log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
|
||||
distance,
|
||||
MATCH_THRESHOLD,
|
||||
identifier.entityPosition(),
|
||||
closestEntity.getRedactionPositionsPerPage()));
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(closestEntity);
|
||||
}
|
||||
|
||||
|
||||
private Map<String, List<RedactionEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
|
||||
|
||||
if (!pageNumbers.stream().allMatch(node::isOnPage)) {
|
||||
throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
|
||||
node,
|
||||
pageNumbers.stream().filter(pageNumber -> !node.isOnPage(pageNumber)).toList(),
|
||||
node.getPages()));
|
||||
}
|
||||
SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
|
||||
|
||||
return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
|
||||
.stream()
|
||||
.map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
|
||||
.collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
|
||||
}
|
||||
|
||||
|
||||
private static boolean allValuesFound(Map<String, List<RedactionEntity>> entitiesByValue, Set<String> entryValues) {
|
||||
|
||||
return entitiesByValue.keySet().equals(entryValues);
|
||||
}
|
||||
|
||||
|
||||
private static boolean pagesMatch(RedactionEntity entity, List<RectangleWithPage> originalPositions) {
|
||||
|
||||
Set<Integer> entityPageNumbers = entity.getRedactionPositionsPerPage().stream().map(RedactionPosition::getPage).map(Page::getNumber).collect(Collectors.toSet());
|
||||
Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
|
||||
return entityPageNumbers.containsAll(originalPageNumbers);
|
||||
}
|
||||
|
||||
|
||||
private double calculateMinDistance(List<RectangleWithPage> originalPositions, RedactionEntity entity) {
|
||||
|
||||
if (originalPositions.size() != countRectangles(entity)) {
|
||||
return Double.MAX_VALUE;
|
||||
}
|
||||
return originalPositions.stream()
|
||||
.mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
|
||||
.sum();
|
||||
}
|
||||
|
||||
|
||||
private static long countRectangles(RedactionEntity entity) {
|
||||
|
||||
return entity.getRedactionPositionsPerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
|
||||
}
|
||||
|
||||
|
||||
private double calculateMinDistancePerRectangle(RedactionEntity entity, int pageNumber, Rectangle2D originalRectangle) {
|
||||
|
||||
return entity.getRedactionPositionsPerPage()
|
||||
.stream()
|
||||
.filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
|
||||
.map(RedactionPosition::getRectanglePerLine)
|
||||
.flatMap(Collection::stream)
|
||||
.mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
|
||||
.min()
|
||||
.orElse(Double.MAX_VALUE);
|
||||
}
|
||||
|
||||
|
||||
private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
|
||||
|
||||
// mirrored coordinates safe comparison
|
||||
double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
|
||||
double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
|
||||
double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
|
||||
double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
|
||||
|
||||
double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
|
||||
double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
|
||||
double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
|
||||
double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
|
||||
|
||||
return Math.abs(minX1 - minX2) //
|
||||
+ Math.abs(minY1 - minY2) //
|
||||
+ Math.abs(maxX1 - maxX2) //
|
||||
+ Math.abs(maxY1 - maxY2);
|
||||
}
|
||||
|
||||
|
||||
private record EntityIdentifier(
|
||||
String value,
|
||||
List<RectangleWithPage> entityPosition,
|
||||
String ruleIdentifier,
|
||||
String reason,
|
||||
String legalBasis,
|
||||
String type,
|
||||
EntityType entityType,
|
||||
boolean redacted,
|
||||
boolean isDictionaryEntry,
|
||||
boolean isDossierDictionaryEntry) {
|
||||
|
||||
public static EntityIdentifier fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
|
||||
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
|
||||
return new EntityIdentifier(redactionLogEntry.getValue(),
|
||||
rectangleWithPages,
|
||||
ruleIdentifier,
|
||||
redactionLogEntry.getReason(),
|
||||
redactionLogEntry.getLegalBasis(),
|
||||
redactionLogEntry.getType(),
|
||||
redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
|
||||
redactionLogEntry.isRedacted(),
|
||||
redactionLogEntry.isDictionaryEntry(),
|
||||
redactionLogEntry.isDossierDictionaryEntry());
|
||||
}
|
||||
|
||||
|
||||
public static EntityIdentifier fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
|
||||
return new EntityIdentifier(manualRedactionEntry.getValue(),
|
||||
rectangleWithPages,
|
||||
"MAN.0.0",
|
||||
manualRedactionEntry.getReason(),
|
||||
manualRedactionEntry.getLegalBasis(),
|
||||
manualRedactionEntry.getType(),
|
||||
EntityType.ENTITY,
|
||||
true,
|
||||
false,
|
||||
false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
|
||||
|
||||
public static RectangleWithPage fromRedactionLogRectangle(Rectangle rectangle) {
|
||||
|
||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||
}
|
||||
|
||||
|
||||
public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(Rectangle rectangle) {
|
||||
|
||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
|
||||
}
|
||||
|
||||
|
||||
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
|
||||
|
||||
return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+3
-3
@@ -1,10 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.adapter;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.AllArgsConstructor;
|
||||
@@ -37,7 +37,7 @@ public class NerEntities {
|
||||
}
|
||||
|
||||
|
||||
public record NerEntity(String value, TextRange textRange, String type) {
|
||||
public record NerEntity(String value, Boundary boundary, String type) {
|
||||
|
||||
}
|
||||
|
||||
+22
-23
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.service.document;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.adapter;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
@@ -9,11 +9,10 @@ import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
|
||||
import lombok.AccessLevel;
|
||||
import lombok.experimental.FieldDefaults;
|
||||
@@ -45,7 +44,7 @@ public class NerEntitiesAdapter {
|
||||
|
||||
return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(document),
|
||||
nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
|
||||
new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
|
||||
new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
|
||||
nerEntityModel.getType())).toList());
|
||||
}
|
||||
|
||||
@@ -64,17 +63,17 @@ public class NerEntitiesAdapter {
|
||||
* @param allowDuplicates allow combining multiple parts of same type
|
||||
* @return A Stream of the combined boundaries
|
||||
*/
|
||||
public Stream<TextRange> combineNerEntities(NerEntities nerEntities,
|
||||
Set<String> essentialTypes,
|
||||
Set<String> typesToCombine,
|
||||
int maxDistanceBetweenParts,
|
||||
int minPartsToCombine,
|
||||
boolean allowDuplicates) {
|
||||
public Stream<Boundary> combineNerEntities(NerEntities nerEntities,
|
||||
Set<String> essentialTypes,
|
||||
Set<String> typesToCombine,
|
||||
int maxDistanceBetweenParts,
|
||||
int minPartsToCombine,
|
||||
boolean allowDuplicates) {
|
||||
|
||||
List<NerEntities.NerEntity> sortedEntities = nerEntities.getNerEntityList()
|
||||
.stream()
|
||||
.filter(entity -> typesToCombine.contains(entity.type()))
|
||||
.sorted(Comparator.comparingInt(entity -> entity.textRange().start()))
|
||||
.sorted(Comparator.comparingInt(entity -> entity.boundary().start()))
|
||||
.toList();
|
||||
|
||||
if (sortedEntities.isEmpty()) {
|
||||
@@ -87,20 +86,20 @@ public class NerEntitiesAdapter {
|
||||
for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) {
|
||||
List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
|
||||
entityClusters.add(currentCluster);
|
||||
int lastEndOffset = startEntity.textRange().end();
|
||||
int lastEndOffset = startEntity.boundary().end();
|
||||
|
||||
for (NerEntities.NerEntity entity : sortedEntities) {
|
||||
if (entity.textRange().start() < lastEndOffset) {
|
||||
if (entity.boundary().start() < lastEndOffset) {
|
||||
continue;
|
||||
}
|
||||
if (distanceIsLargerThanMaxDistance(lastEndOffset, entity, maxDistanceBetweenParts) || isDuplicate(currentCluster, entity, allowDuplicates)) {
|
||||
currentCluster = new LinkedList<>();
|
||||
entityClusters.add(currentCluster);
|
||||
currentCluster.add(entity);
|
||||
lastEndOffset = entity.textRange().end();
|
||||
lastEndOffset = entity.boundary().end();
|
||||
} else {
|
||||
currentCluster.add(entity);
|
||||
lastEndOffset = entity.textRange().end();
|
||||
lastEndOffset = entity.boundary().end();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,7 +120,7 @@ public class NerEntitiesAdapter {
|
||||
*
|
||||
* @return A Stream of the combined entities of type "CBI_address"
|
||||
*/
|
||||
public Stream<TextRange> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) {
|
||||
public Stream<Boundary> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) {
|
||||
|
||||
return combineNerEntities(entityRecognitionEntities,
|
||||
CBI_ADDRESS_ESSENTIAL_TYPES,
|
||||
@@ -140,13 +139,13 @@ public class NerEntitiesAdapter {
|
||||
|
||||
private static boolean distanceIsLargerThanMaxDistance(int lastEndOffset, NerEntities.NerEntity entity, int maxDistance) {
|
||||
|
||||
return (entity.textRange().start() - lastEndOffset) > maxDistance;
|
||||
return (entity.boundary().start() - lastEndOffset) > maxDistance;
|
||||
}
|
||||
|
||||
|
||||
private static TextRange toContainingBoundary(List<NerEntities.NerEntity> nerEntities) {
|
||||
private static Boundary toContainingBoundary(List<NerEntities.NerEntity> nerEntities) {
|
||||
|
||||
return TextRange.merge(nerEntities.stream().map(NerEntities.NerEntity::textRange).toList());
|
||||
return Boundary.merge(nerEntities.stream().map(NerEntities.NerEntity::boundary).toList());
|
||||
}
|
||||
|
||||
|
||||
@@ -163,7 +162,7 @@ public class NerEntitiesAdapter {
|
||||
|
||||
private static List<Integer> getStringStartOffsetsForMainSections(Document document) {
|
||||
|
||||
return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getTextRange).map(TextRange::start).toList();
|
||||
return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList();
|
||||
}
|
||||
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class TenantRules {
|
||||
|
||||
private Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
|
||||
|
||||
}
|
||||
+23
-37
@@ -1,24 +1,24 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.Patterns;
|
||||
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
@@ -107,53 +107,39 @@ public class Dictionary {
|
||||
if (StringUtils.isEmpty(value)) {
|
||||
throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
|
||||
}
|
||||
boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
|
||||
Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
|
||||
|
||||
String cleanedValue = value;
|
||||
if (isCaseInsensitive) {
|
||||
cleanedValue = cleanedValue.toLowerCase(Locale.US);
|
||||
}
|
||||
localAccessMap.get(type)
|
||||
.getLocalEntriesWithMatchedRules()
|
||||
.merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
|
||||
localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(value.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
|
||||
if (alsoAddLastname) {
|
||||
String lastname = cleanedValue.split(" ")[0];
|
||||
localAccessMap.get(type)
|
||||
.getLocalEntriesWithMatchedRules()
|
||||
.merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
|
||||
String lastname = value.split(" ")[0];
|
||||
localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void recommendEverywhere(TextEntity textEntity) {
|
||||
public void recommendEverywhere(RedactionEntity redactionEntity) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
|
||||
addLocalDictionaryEntry(redactionEntity.getType(), redactionEntity.getValue(), redactionEntity.getMatchedRuleList(), false);
|
||||
}
|
||||
|
||||
|
||||
public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
|
||||
public void recommendEverywhereWithLastNameSeparately(RedactionEntity redactionEntity) {
|
||||
|
||||
addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
|
||||
addLocalDictionaryEntry(redactionEntity.getType(), redactionEntity.getValue(), redactionEntity.getMatchedRuleList(), true);
|
||||
}
|
||||
|
||||
|
||||
public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
|
||||
public void addMultipleAuthorsAsRecommendation(RedactionEntity redactionEntity) {
|
||||
|
||||
splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
|
||||
String cleanedWord = redactionEntity.getValue().replaceAll(",", " ").replaceAll(" ", " ").trim() + " ";
|
||||
Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
|
||||
Matcher matcher = pattern.matcher(cleanedWord);
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static List<String> splitIntoAuthorNames(TextEntity textEntity) {
|
||||
|
||||
List<String> splitAuthorNames;
|
||||
if (textEntity.getValue().contains(",")) {
|
||||
splitAuthorNames = Arrays.asList(textEntity.getValue().split(","));
|
||||
} else {
|
||||
splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group().trim();
|
||||
if (match.length() >= 3) {
|
||||
addLocalDictionaryEntry(redactionEntity.getType(), match, redactionEntity.getMatchedRuleList(), true);
|
||||
}
|
||||
}
|
||||
return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList();
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
+2
-11
@@ -1,15 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.model.dictionary;
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
|
||||
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.MatchedRule;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
@@ -111,11 +109,4 @@ public class DictionaryModel implements Serializable {
|
||||
return falseRecommendationsSearch;
|
||||
}
|
||||
|
||||
public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
|
||||
var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
|
||||
|
||||
return localEntriesWithMatchedRules.get(cleanedValue);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user