Merge branch 'master' into RED-3621

# Conflicts:
#	persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
This commit is contained in:
Philipp Schramm 2022-04-04 11:04:47 +02:00
commit b2f0af3141
20 changed files with 267 additions and 190 deletions

View File

@ -5,7 +5,7 @@
<parent>
<groupId>com.atlassian.bamboo</groupId>
<artifactId>bamboo-specs-parent</artifactId>
<version>8.0.3</version>
<version>8.1.3</version>
<relativePath/>
</parent>

View File

@ -1,10 +1,12 @@
package buildjob;
import java.time.DayOfWeek;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
import java.time.LocalTime;
import com.atlassian.bamboo.specs.api.BambooSpec;
import com.atlassian.bamboo.specs.api.builders.BambooKey;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@ -15,18 +17,18 @@ import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
import com.atlassian.bamboo.specs.api.builders.plan.Stage;
import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
import com.atlassian.bamboo.specs.api.builders.plan.configuration.ConcurrentBuilds;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.builders.task.*;
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
import com.atlassian.bamboo.specs.util.BambooServer;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
/**
* Plan configuration for Bamboo.
* Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
@ -75,10 +77,7 @@ public class PlanSpec {
}
public Plan createPlanBuild() {
return new Plan(
project(),
SERVICE_NAME, new BambooKey(SERVICE_KEY))
.description("Build Plan for Persitence Service")
return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Build Plan for Persitence Service").variables(new Variable("maven_add_param", ""))
.stages(new Stage("Default Stage")
.jobs(new Job("Default Job",
new BambooKey("JOB1"))
@ -87,9 +86,7 @@ public class PlanSpec {
.description("Clean")
.inlineBody("#!/bin/bash\n" +
"set -e\n" +
"rm -rf ./*"),
new VcsCheckoutTask()
.description("Checkout Default Repository")
"rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true)
.checkoutItems(new CheckoutItem().defaultRepository()),
new ScriptTask()
.description("Build")
@ -109,9 +106,7 @@ public class PlanSpec {
.description("${bamboo.g.gitTag}")
.tagName("${bamboo.g.gitTag}")
.defaultRepository())
.dockerConfiguration(
new DockerConfiguration()
.image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
.volume("/var/run/docker.sock", "/var/run/docker.sock")
@ -140,29 +135,22 @@ public class PlanSpec {
.description("Clean")
.inlineBody("#!/bin/bash\n" +
"set -e\n" +
"rm -rf ./*"),
new VcsCheckoutTask()
.description("Checkout Default Repository")
"rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true)
.checkoutItems(new CheckoutItem().defaultRepository()),
new ScriptTask()
.description("Sonar")
.location(Location.FILE)
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
.argument(SERVICE_NAME))
.dockerConfiguration(
new DockerConfiguration()
.image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
.volume("/var/run/docker.sock", "/var/run/docker.sock")
)
)
)
.linkedRepositories("RED / " + SERVICE_NAME)
.triggers(
new ScheduledTrigger()
.scheduleOnceDaily(LocalTime.of(12, 00)),
new BitbucketServerTrigger())
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
.planBranchManagement(new PlanBranchManagement()
.createForVcsBranchMatching("release.*")
.notificationForCommitters());

View File

@ -3,49 +3,58 @@ set -e
SERVICE_NAME=$1
if [[ "${bamboo_version_tag}" = "dev" ]]
then
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
if [[ "$bamboo_planRepository_branchName" == "master" ]]
then
branchVersion=$(cat pom.xml | grep -Eo "<version>.*" | sed -s 's|<version>\(.*\)\..*\(-*.*\)</version>|\1|')
latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
newVersion="$(semver $latestVersion -p -i minor)"
elif [[ "$bamboo_planRepository_branchName" == release* ]]
then
branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
latestVersion=$( semver $(git tag -l $branchVersion.*) | tail -n1)
newVersion="$(semver $latestVersion -p -i patch)"
elif [[ "${bamboo_version_tag}" != "dev" ]]
then
newVersion="${bamboo_version_tag}"
else
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
--no-transfer-progress \
${bamboo_maven_add_param} \
clean install \
-Djava.security.egd=file:/dev/./urandomelse
else
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
versions:set \
-DnewVersion=${bamboo_version_tag}
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
versions:set \
-DnewVersion=${bamboo_version_tag}
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
--no-transfer-progress \
clean deploy \
-e \
-DdeployAtEnd=true \
-Dmaven.wagon.http.ssl.insecure=true \
-Dmaven.wagon.http.ssl.allowall=true \
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
exit 0
fi
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
echo "gitTag=${newVersion}" > git.tag
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
versions:set \
-DnewVersion=${newVersion}
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
versions:set \
-DnewVersion=${newVersion}
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
--no-transfer-progress \
clean deploy \
${bamboo_maven_add_param} \
-e \
-DdeployAtEnd=true \
-Dmaven.wagon.http.ssl.insecure=true \
-Dmaven.wagon.http.ssl.allowall=true \
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/red-platform-releases
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
package
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-image-v1/pom.xml \
docker:push
if [[ "${bamboo_version_tag}" = "dev" ]]
then
echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
else
echo "gitTag=${bamboo_version_tag}" > git.tag
fi

View File

@ -4,46 +4,41 @@ set -e
SERVICE_NAME=$1
echo "build jar binaries"
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
mvn -f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
--no-transfer-progress \
clean install \
-Djava.security.egd=file:/dev/./urandomelse
echo "dependency-check:aggregate"
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
org.owasp:dependency-check-maven:aggregate
if [[ -z "${bamboo_repository_pr_key}" ]]
then
echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
sonar:sonar \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
sonar:sonar \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
else
echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
--no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
sonar:sonar \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
mvn --no-transfer-progress \
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
sonar:sonar \
-Dsonar.projectKey=RED_$SERVICE_NAME \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-Dsonar.dependencyCheck.jsonReportPath=target/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=target/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=target/dependency-check-report.html
fi

View File

@ -1,17 +1,18 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@ -61,6 +62,7 @@ public class FileModel {
private String dossierId;
private OffsetDateTime redactionModificationDate;
private OffsetDateTime fileManipulationDate;
private boolean hasHighlights;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;

View File

@ -3,7 +3,7 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d
import lombok.Getter;
public enum FileType {
UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json");
UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json");
@Getter
private final String extension;

View File

@ -1,17 +1,25 @@
package com.iqser.red.service.persistence.service.v1.api.resources;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
@ResponseStatus(value = HttpStatus.OK)
public interface DossierResource {
@ -52,8 +60,9 @@ public interface DossierResource {
List<Dossier> getAllDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
@GetMapping(value = REST_PATH + INFO_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
DossierInformation getDossierInformation();
DossierInformation getDossierInformation(@RequestBody List<String> filteredDossierIds);
@GetMapping(value = REST_PATH + DOSSIER_ID_PATH_PARAM, produces = MediaType.APPLICATION_JSON_VALUE)
Dossier getDossierById(@PathVariable(DOSSIER_ID_PARAM) String dossierId,

View File

@ -11,6 +11,7 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
@ -20,6 +21,7 @@ public interface ReanalysisResource {
String REANALYZE_PATH = "/reanalyze";
String IMPORT_REDACTIONS_PATH = "/import-redactions";
String TEXT_HIGHLIGHT_CONVERSION_PATH = "/texthighlights-conversion";
String CONVERT_TEXT_HIGHLIGHTS_PATH = "/convert-texthighlights";
String OCR_REANALYZE_PATH = "/ocr/reanalyze";
String REINDEX_PATH = "/reindex";
String DOSSIER_ID_PARAM = "dossierId";
@ -65,4 +67,8 @@ public interface ReanalysisResource {
@PostMapping(value = TEXT_HIGHLIGHT_CONVERSION_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest);
@PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest);
}

View File

@ -1,22 +1,35 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@ -138,6 +151,9 @@ public class FileEntity {
@Column
private OffsetDateTime lastManualChangeDate;
@Column
private boolean hasHighlights;
@Column(columnDefinition = "text", name = "excluded_pages")
@Convert(converter = JSONIntegerSetConverter.class)
private Set<Integer> excludedPages = new HashSet<>();

View File

@ -1,5 +1,17 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -9,17 +21,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
@ -31,7 +34,7 @@ public class FileStatusPersistenceService {
private final DossierPersistenceService dossierService;
public void createStatus(String dossierId, String fileId, String filename, String uploader) {
public void createStatus(String dossierId, String fileId, String filename, String uploader, boolean hasHighlights) {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
FileEntity file = new FileEntity();
@ -46,6 +49,7 @@ public class FileStatusPersistenceService {
file.setLastUploaded(now);
file.setLastUpdated(now);
file.setFileManipulationDate(now);
file.setHasHighlights(hasHighlights);
fileRepository.save(file);
}
@ -295,14 +299,14 @@ public class FileStatusPersistenceService {
@Transactional
public void overwriteFile(String fileId, String uploader, String filename) {
public void overwriteFile(String fileId, String uploader, String filename, boolean hasHighlights) {
fileRepository.findById(fileId).ifPresent((file) -> {
file.setExcludedPages(new HashSet<>());
});
int countUpdate = fileRepository.overwriteFile(fileId, filename, uploader, ProcessingStatus.FULLREPROCESS, WorkflowStatus.NEW, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), hasHighlights);
if (countUpdate == 0) {
throw new NotFoundException("Unknown file=" + fileId);
}
@ -354,14 +358,24 @@ public class FileStatusPersistenceService {
return fileRepository.countSoftDeletedFiles(dossierId);
}
public List<FileEntity> getAllRelevantStatusesForReanalysisScheduler() {
return fileRepository.getAllRelevantStatusesForReanalysisScheduler();
}
@Transactional
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) {
fileRepository.updateFileModificationDate(fileId, fileManipulationDate);
}
@Transactional
public void updateHasHighlights(String fileId, boolean hasHighlights) {
fileRepository.updateHasHighlights(fileId, hasHighlights);
}
}

View File

@ -1,15 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.time.OffsetDateTime;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import java.time.OffsetDateTime;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
public interface FileRepository extends JpaRepository<FileEntity, String> {
@ -67,25 +67,31 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
"f.lastIndexed = :lastIndexed where f.id = :fileId")
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
OffsetDateTime lastIndexed);
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastOCRTime = :lastOCRTime where f.id = :fileId")
void updateLastOCRTime(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastOCRTime);
@Modifying
@Query("update FileEntity f set f.fileManipulationDate = :fileManipulationDate where f.id = :fileId")
void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate);
@Modifying
@Query("update FileEntity f set f.hasHighlights = :hasHighlights where f.id = :fileId")
void updateHasHighlights(String fileId, boolean hasHighlights);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId")
void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments);
@Modifying
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " +
"f.deleted = :softDeletedTime where f.id = :fileId")
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId")
int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
OffsetDateTime softDeletedTime);
@ -113,17 +119,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId")
int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated);
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " +
"f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
"f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
"f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
"f.deleted = null, f.hardDeletedTime = null " +
"where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus,
WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated);
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
OffsetDateTime lastUpdated, boolean hasHighlights);
@Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null")
int countSoftDeletedFiles(String dossierId);

View File

@ -107,10 +107,15 @@ public class DossierController implements DossierResource {
}
@Override
public DossierInformation getDossierInformation() {
public DossierInformation getDossierInformation(@RequestBody List<String> filteredDossierIds) {
DossierInformation dossierInformation = new DossierInformation();
dossierService.getAllDossiers().forEach(d -> {
var dossiers = dossierService.getAllDossiers();
if (filteredDossierIds != null && filteredDossierIds.size() > 0) {
dossiers = dossiers.stream().filter(d -> filteredDossierIds.contains(d.getId())).collect(Collectors.toList());
}
dossiers.forEach(d -> {
if (d.getHardDeletedTime() != null) {
dossierInformation.setNumberOfHardDeletedDossiers(dossierInformation.getNumberOfHardDeletedDossiers() + 1);
} else if (d.getSoftDeletedTime() != null) {

View File

@ -17,12 +17,13 @@ import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.IndexingService;
import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -83,10 +84,23 @@ public class ReanalysisController implements ReanalysisResource {
}
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
indexingService.reindex(dossierId, fileIds, dropIndex);
dossierPersistenceService.getAndValidateDossier(dossierId);
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
if (dossierFile.getLastOCRTime() != null) {
throw new ConflictException("File already has been OCR processed");
}
ocrFiles(dossierId, Sets.newHashSet(fileId));
}
}
@ -109,6 +123,13 @@ public class ReanalysisController implements ReanalysisResource {
}
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
indexingService.reindex(dossierId, fileIds, dropIndex);
}
public TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest) {
var textHighlightResponse = pDFTronRedactionClient.processTextHighlights(textHighlightRequest);
@ -123,22 +144,15 @@ public class ReanalysisController implements ReanalysisResource {
}
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
public void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest) {
dossierPersistenceService.getAndValidateDossier(dossierId);
boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest);
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
if (dossierFile.getLastOCRTime() != null) {
throw new ConflictException("File already has been OCR processed");
}
ocrFiles(dossierId, Sets.newHashSet(fileId));
fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights);
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true);
}
}

View File

@ -4,15 +4,12 @@ import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -27,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.model.IndexMessageType;
@ -78,8 +76,9 @@ public class FileService {
}
}
boolean hasHighlights;
try {
pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData()));
hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData()));
} catch (FeignException e) {
log.warn("Failed to optimize file: {}", request.getFilename(), e);
@ -89,11 +88,11 @@ public class FileService {
if (existingStatus != null) {
// the file is already uploaded, just reanalyse it.
fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename());
fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
} else {
// the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename());
fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
}
return new JSONPrimitive<>(fileId);
}

View File

@ -308,9 +308,9 @@ public class FileStatusService {
@Transactional
public void createStatus(String dossierId, String fileId, String uploader, String filename) {
public void createStatus(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader);
fileStatusPersistenceService.createStatus(dossierId, fileId, filename, uploader, hasHighlights);
addToAnalysisQueue(dossierId, fileId, false, Set.of());
}
@ -351,11 +351,18 @@ public class FileStatusService {
}
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate){
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) {
fileStatusPersistenceService.updateFileModificationDate(fileId, fileManipulationDate);
}
public void updateHasHighlights(String fileId, boolean hasHighlights) {
fileStatusPersistenceService.updateHasHighlights(fileId, hasHighlights);
}
public void setAssignee(String fileId, String assignee) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -443,9 +450,9 @@ public class FileStatusService {
@Transactional
public void overwriteFile(String dossierId, String fileId, String uploader, String filename) {
public void overwriteFile(String dossierId, String fileId, String uploader, String filename, boolean hasHighlights) {
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename);
fileStatusPersistenceService.overwriteFile(fileId, uploader, filename, hasHighlights);
wipeFileData(dossierId, fileId);
setStatusFullReprocess(dossierId, fileId, false);
}

View File

@ -0,0 +1,11 @@
databaseChangeLog:
- changeSet:
id: added-has-highlights-to-file
author: dom
changes:
- addColumn:
columns:
- column:
name: has_highlights
type: BOOLEAN
tableName: file

View File

@ -44,4 +44,6 @@ databaseChangeLog:
- include:
file: db/changelog/18-add-migration-table.yaml
- include:
file: db/changelog/19-add-index-information-table.yaml
file: db/changelog/19-added-has-highlights-to-file.changelog.yaml
- include:
file: db/changelog/20-add-index-information-table.yaml

View File

@ -113,7 +113,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThatThrownBy(() -> dossierClient.undeleteDossiers(Sets.newHashSet(dossier.getId()))).isInstanceOf(FeignException.class);
var dossierInformation = dossierClient.getDossierInformation();
var dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(1);
@ -147,7 +147,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getArchivedDossiers()).hasSize(1);
assertThat(dossierClient.getSoftDeletedDossiers()).hasSize(0);
var dossierInformation = dossierClient.getDossierInformation();
var dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(1);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(0);
@ -158,7 +158,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getAllDossiers(false, false)).hasSize(2);
assertThat(dossierClient.getArchivedDossiers()).hasSize(0);
dossierInformation = dossierClient.getDossierInformation();
dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(2);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(0);
@ -171,7 +171,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getArchivedDossiers()).hasSize(2);
assertThat(dossierClient.getSoftDeletedDossiers()).hasSize(0);
dossierInformation = dossierClient.getDossierInformation();
dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(0);
@ -187,7 +187,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getArchivedDossiers()).hasSize(1);
assertThat(dossierClient.getSoftDeletedDossiers()).hasSize(1);
dossierInformation = dossierClient.getDossierInformation();
dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(1);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(0);
@ -201,7 +201,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getArchivedDossiers()).hasSize(0);
assertThat(dossierClient.getSoftDeletedDossiers()).hasSize(1);
dossierInformation = dossierClient.getDossierInformation();
dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(1);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(1);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(0);
@ -217,7 +217,7 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
assertThat(dossierClient.getArchivedDossiers()).hasSize(1);
assertThat(dossierClient.getSoftDeletedDossiers()).hasSize(0);
dossierInformation = dossierClient.getDossierInformation();
dossierInformation = dossierClient.getDossierInformation(List.of());
assertThat(dossierInformation.getNumberOfActiveDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfSoftDeletedDossiers()).isEqualTo(0);
assertThat(dossierInformation.getNumberOfHardDeletedDossiers()).isEqualTo(1);

View File

@ -27,7 +27,7 @@
<properties>
<redaction-service.version>3.90.0</redaction-service.version>
<search-service.version>2.26.0</search-service.version>
<pdftron-redaction-service.version>3.53.0</pdftron-redaction-service.version>
<pdftron-redaction-service.version>3.59.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.19.0</redaction-report-service.version>
</properties>