Pull request #332: RED-3621: Implemented IndexInformation

Merge in RED/persistence-service from RED-3621 to master

* commit 'b2f0af3141decfd6b5caf151beb1b0d9db3cccf9':
  RED-3621: Implemented IndexInformation
This commit is contained in:
Philipp Schramm 2022-04-04 11:32:54 +02:00
commit fa88531148
23 changed files with 462 additions and 162 deletions

View File

@ -1,10 +1,12 @@
package buildjob;
import java.time.DayOfWeek;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
import java.time.LocalTime;
import com.atlassian.bamboo.specs.api.BambooSpec;
import com.atlassian.bamboo.specs.api.builders.BambooKey;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
@ -15,20 +17,18 @@ import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
import com.atlassian.bamboo.specs.api.builders.plan.Stage;
import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
import com.atlassian.bamboo.specs.api.builders.plan.configuration.ConcurrentBuilds;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.builders.task.*;
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
import com.atlassian.bamboo.specs.builders.trigger.RepositoryPollingTrigger;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
import com.atlassian.bamboo.specs.util.BambooServer;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
/**
* Plan configuration for Bamboo.
* Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
@ -77,11 +77,7 @@ public class PlanSpec {
}
public Plan createPlanBuild() {
return new Plan(
project(),
SERVICE_NAME, new BambooKey(SERVICE_KEY))
.description("Build Plan for Persitence Service")
.variables(new Variable("maven_add_param", ""))
return new Plan(project(), SERVICE_NAME, new BambooKey(SERVICE_KEY)).description("Build Plan for Persitence Service").variables(new Variable("maven_add_param", ""))
.stages(new Stage("Default Stage")
.jobs(new Job("Default Job",
new BambooKey("JOB1"))
@ -90,10 +86,7 @@ public class PlanSpec {
.description("Clean")
.inlineBody("#!/bin/bash\n" +
"set -e\n" +
"rm -rf ./*"),
new VcsCheckoutTask()
.description("Checkout Default Repository")
.cleanCheckout(true)
"rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true)
.checkoutItems(new CheckoutItem().defaultRepository()),
new ScriptTask()
.description("Build")
@ -113,9 +106,7 @@ public class PlanSpec {
.description("${bamboo.g.gitTag}")
.tagName("${bamboo.g.gitTag}")
.defaultRepository())
.dockerConfiguration(
new DockerConfiguration()
.image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
.volume("/var/run/docker.sock", "/var/run/docker.sock")
@ -144,29 +135,22 @@ public class PlanSpec {
.description("Clean")
.inlineBody("#!/bin/bash\n" +
"set -e\n" +
"rm -rf ./*"),
new VcsCheckoutTask()
.description("Checkout Default Repository")
.cleanCheckout(true)
"rm -rf ./*"), new VcsCheckoutTask().description("Checkout Default Repository").cleanCheckout(true)
.checkoutItems(new CheckoutItem().defaultRepository()),
new ScriptTask()
.description("Sonar")
.location(Location.FILE)
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-java.sh")
.argument(SERVICE_NAME))
.dockerConfiguration(
new DockerConfiguration()
.image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
.dockerConfiguration(new DockerConfiguration().image("nexus.iqser.com:5001/infra/maven:3.8.4-openjdk-17-slim")
.dockerRunArguments("--net=host")
.volume("/etc/maven/settings.xml", "/usr/share/maven/conf/settings.xml")
.volume("/var/run/docker.sock", "/var/run/docker.sock")
)
)
)
.linkedRepositories("RED / " + SERVICE_NAME)
.triggers(
new ScheduledTrigger()
.scheduleOnceDaily(LocalTime.of(23, 00)))
.triggers(new ScheduledTrigger().scheduleOnceDaily(LocalTime.of(23, 00)))
.planBranchManagement(new PlanBranchManagement()
.createForVcsBranchMatching("release.*")
.notificationForCommitters());

View File

@ -1,17 +1,18 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor

View File

@ -0,0 +1,19 @@
package com.iqser.red.service.persistence.service.v1.api.model.index;
import java.time.OffsetDateTime;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class IndexInformation {
private String indexConfigurationHash;
private OffsetDateTime updateDate;
}

View File

@ -1,17 +1,25 @@
package com.iqser.red.service.persistence.service.v1.api.resources;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierChange;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierInformation;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
@ResponseStatus(value = HttpStatus.OK)
public interface DossierResource {
@ -52,6 +60,7 @@ public interface DossierResource {
List<Dossier> getAllDossiers(@RequestParam(name = INCLUDE_ARCHIVED_PARAM, defaultValue = "false", required = false) boolean includeArchived,
@RequestParam(name = INCLUDE_DELETED_PARAM, defaultValue = "false", required = false) boolean includeDeleted);
@GetMapping(value = REST_PATH + INFO_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
DossierInformation getDossierInformation(@RequestBody List<String> filteredDossierIds);

View File

@ -0,0 +1,29 @@
package com.iqser.red.service.persistence.service.v1.api.resources;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation;
public interface IndexInformationResource {
String PATH = "/index-information";
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@GetMapping(value = PATH, produces = MediaType.APPLICATION_JSON_VALUE)
IndexInformation getIndexInformation();
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void updateIndexInformation(@RequestBody IndexInformation indexInformation);
}

View File

@ -67,6 +67,7 @@ public interface ReanalysisResource {
@PostMapping(value = TEXT_HIGHLIGHT_CONVERSION_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest);
@PostMapping(value = CONVERT_TEXT_HIGHLIGHTS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void convertTextHighlights(@RequestBody TextHighlightConversionRequest textHighlightRequest);

View File

@ -1,14 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.audit.AuditEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
import org.springframework.boot.autoconfigure.domain.EntityScan;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.cloud.openfeign.support.PageJacksonModule;
@ -18,9 +9,20 @@ import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.annotations.CommentEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.audit.AuditEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.index.IndexInformationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.migration.MigrationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.notification.NotificationEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ColorsRepository;
@Configuration
@ComponentScan
@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class, MigrationEntity.class})
@EntityScan(basePackageClasses = {CommentEntity.class, AuditEntity.class, NotificationEntity.class, ColorsEntity.class, DossierEntity.class, DownloadStatusEntity.class, MigrationEntity.class, IndexInformationEntity.class})
@EnableJpaRepositories(basePackageClasses = ColorsRepository.class)
@EnableFeignClients(basePackageClasses = {PDFTronRedactionClient.class})
public class PersistenceServiceProcessorConfiguration {

View File

@ -1,22 +1,35 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Convert;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor

View File

@ -0,0 +1,37 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.index;
import java.time.OffsetDateTime;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import com.iqser.red.service.persistence.service.v1.api.utils.SuppressFBWarnings;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Entity
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "index_information")
@SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
public class IndexInformationEntity {
public final static String ID = "CONFIG_ID";
@Id
private final String id = IndexInformationEntity.ID;
@Column
private String indexConfigurationHash;
@Column
private OffsetDateTime updateDate;
}

View File

@ -1,5 +1,17 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
@ -9,17 +21,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
@ -355,18 +358,23 @@ public class FileStatusPersistenceService {
return fileRepository.countSoftDeletedFiles(dossierId);
}
public List<FileEntity> getAllRelevantStatusesForReanalysisScheduler() {
return fileRepository.getAllRelevantStatusesForReanalysisScheduler();
}
@Transactional
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) {
fileRepository.updateFileModificationDate(fileId, fileManipulationDate);
}
@Transactional
public void updateHasHighlights(String fileId, boolean hasHighlights){
public void updateHasHighlights(String fileId, boolean hasHighlights) {
fileRepository.updateHasHighlights(fileId, hasHighlights);
}

View File

@ -0,0 +1,32 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence;
import java.util.Optional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.index.IndexInformationEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class IndexInformationPersistenceService {
private final IndexInformationRepository indexInformationRepository;
public IndexInformationEntity getIndexInformation() {
Optional<IndexInformationEntity> indexInformation = indexInformationRepository.findById(IndexInformationEntity.ID);
return indexInformation.orElseThrow(() -> new NotFoundException("Index information does not exist"));
}
public void updateIndexInformation(IndexInformationEntity indexInformation) {
indexInformationRepository.save(indexInformation);
}
}

View File

@ -1,15 +1,15 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.time.OffsetDateTime;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import java.time.OffsetDateTime;
import java.util.List;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
public interface FileRepository extends JpaRepository<FileEntity, String> {
@ -67,29 +67,31 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated," +
"f.lastIndexed = :lastIndexed where f.id = :fileId")
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
OffsetDateTime lastIndexed);
void setUpdateStatusIndexingSuccessful(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated, OffsetDateTime lastIndexed);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.lastOCRTime = :lastOCRTime where f.id = :fileId")
void updateLastOCRTime(String fileId, OffsetDateTime lastUpdated, OffsetDateTime lastOCRTime);
@Modifying
@Query("update FileEntity f set f.fileManipulationDate = :fileManipulationDate where f.id = :fileId")
void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate);
@Modifying
@Query("update FileEntity f set f.hasHighlights = :hasHighlights where f.id = :fileId")
void updateHasHighlights(String fileId, boolean hasHighlights);
@Modifying
@Query("update FileEntity f set f.lastUpdated = :lastUpdated, f.hasAnnotationComments = :hasAnnotationComments where f.id = :fileId")
void updateHasComments(String fileId, OffsetDateTime lastUpdated, boolean hasAnnotationComments);
@Modifying
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " +
"f.deleted = :softDeletedTime where f.id = :fileId")
@Query("update FileEntity f set f.processingStatus = :processingStatus, f.lastUpdated = :lastUpdated, " + "f.deleted = :softDeletedTime where f.id = :fileId")
int setSoftDelete(String fileId, ProcessingStatus processingStatus, OffsetDateTime lastUpdated,
OffsetDateTime softDeletedTime);
@ -117,17 +119,11 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Query("update FileEntity f set f.excludedFromAutomaticAnalysis = :excludedFromAutomaticAnalysis, f.lastUpdated = :lastUpdated where f.id = :fileId")
int toggleAutomaticAnalysis(String fileId, boolean excludedFromAutomaticAnalysis, OffsetDateTime lastUpdated);
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " +
"f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " +
"f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " +
"f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," +
"f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " +
"f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " +
"f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " +
"where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus,
WorkflowStatus workflowStatus, OffsetDateTime lastUploaded, OffsetDateTime lastUpdated, boolean hasHighlights);
@Query("update FileEntity f set f.filename = :filename, f.uploader = :uploader, f.processingStatus = :processingStatus, " + "f.workflowStatus = :workflowStatus, f.lastUploaded = :lastUploaded, f.lastUpdated = :lastUpdated, f.fileManipulationDate = :lastUploaded, " + "f.lastOCRTime = null, f.excluded = false, f.lastProcessed = null, f.lastReviewer = null, f.lastApprover = null, " + "f.assignee = null, f.approvalDate = null, f.numberOfAnalyses = 0, f.lastManualChangeDate = null, f.redactionModificationDate = null," + "f.dictionaryVersion = 0, f.dossierDictionaryVersion = 0, f.rulesVersion = 0, f.hasImages = false, " + "f.hasHints = false, f.hasRedactions = false, f.hasSuggestions = false, f.hasUpdates = false, " + "f.deleted = null, f.hardDeletedTime = null, f.hasHighlights = :hasHighlights " + "where f.id = :fileId")
int overwriteFile(String fileId, String filename, String uploader, ProcessingStatus processingStatus, WorkflowStatus workflowStatus, OffsetDateTime lastUploaded,
OffsetDateTime lastUpdated, boolean hasHighlights);
@Query("select count(f) from FileEntity f where f.dossierId = :dossierId and f.deleted is not null and f.hardDeletedTime is null")
int countSoftDeletedFiles(String dossierId);

View File

@ -0,0 +1,9 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import com.iqser.red.service.persistence.management.v1.processor.entity.index.IndexInformationEntity;
public interface IndexInformationRepository extends JpaRepository<IndexInformationEntity, String> {
}

View File

@ -27,6 +27,10 @@
<groupId>com.iqser.red.service</groupId>
<artifactId>pdftron-redaction-service-api-v1</artifactId>
</exclusion>
<exclusion>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-api-v1</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>

View File

@ -0,0 +1,35 @@
package com.iqser.red.service.peristence.v1.server.controller;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.persistence.management.v1.processor.entity.index.IndexInformationEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.IndexInformationPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation;
import com.iqser.red.service.persistence.service.v1.api.resources.IndexInformationResource;
import lombok.RequiredArgsConstructor;
@RestController
@RequiredArgsConstructor
public class IndexInformationController implements IndexInformationResource {
private final IndexInformationPersistenceService indexInformationPersistenceService;
@Override
public IndexInformation getIndexInformation() {
return convert(indexInformationPersistenceService.getIndexInformation(), IndexInformation.class);
}
@Override
public void updateIndexInformation(@RequestBody IndexInformation indexInformation) {
indexInformationPersistenceService.updateIndexInformation(convert(indexInformation, IndexInformationEntity.class));
}
}

View File

@ -24,7 +24,6 @@ import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.IndexingService;
import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -49,8 +48,7 @@ public class ReanalysisController implements ReanalysisResource {
@Override
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
public void reanalyzeDossier(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
var relevantFiles = getAllFilesForDossier(dossierId, true);
reanalyseFiles(dossierId, force, relevantFiles);
@ -86,11 +84,23 @@ public class ReanalysisController implements ReanalysisResource {
}
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex,
@RequestBody Set<String> fileIds) {
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
indexingService.reindex(dossierId, fileIds, dropIndex);
dossierPersistenceService.getAndValidateDossier(dossierId);
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
if (dossierFile.getLastOCRTime() != null) {
throw new ConflictException("File already has been OCR processed");
}
ocrFiles(dossierId, Sets.newHashSet(fileId));
}
}
@ -113,15 +123,19 @@ public class ReanalysisController implements ReanalysisResource {
}
public void reindex(@RequestParam(value = DOSSIER_ID_PARAM, required = false) String dossierId,
@RequestParam(value = "dropIndex", required = false, defaultValue = FALSE) boolean dropIndex, @RequestBody Set<String> fileIds) {
indexingService.reindex(dossierId, fileIds, dropIndex);
}
public TextHighlightResponse processTextHighlights(@RequestBody TextHighlightRequest textHighlightRequest) {
var textHighlightResponse = pDFTronRedactionClient.processTextHighlights(textHighlightRequest);
if (textHighlightRequest.getOperation()
.equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation()
.equals(TextHighlightOperation.CONVERT)) {
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation().equals(TextHighlightOperation.REMOVE) || textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) {
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}
if (textHighlightRequest.getOperation().equals(TextHighlightOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true);
@ -135,8 +149,7 @@ public class ReanalysisController implements ReanalysisResource {
boolean hasTextHighlights = pDFTronRedactionClient.convertTextHighlights(textHighlightRequest);
fileStatusService.updateHasHighlights(textHighlightRequest.getFileId(), hasTextHighlights);
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS));
fileStatusService.updateFileModificationDate(textHighlightRequest.getFileId(), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
if (textHighlightRequest.getOperation().equals(TextHighlightConversionOperation.CONVERT)) {
fileStatusService.setStatusFullReprocess(textHighlightRequest.getDossierId(), textHighlightRequest.getFileId(), true);
@ -144,27 +157,6 @@ public class ReanalysisController implements ReanalysisResource {
}
public void ocrFile(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID_PARAM) String fileId,
@RequestParam(value = "force", required = false, defaultValue = FALSE) boolean force) {
dossierPersistenceService.getAndValidateDossier(dossierId);
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus()
.equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
if (dossierFile.getLastOCRTime() != null) {
throw new ConflictException("File already has been OCR processed");
}
ocrFiles(dossierId, Sets.newHashSet(fileId));
}
}
private List<FileModel> getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);

View File

@ -4,15 +4,12 @@ import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
@ -27,6 +24,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.search.v1.model.IndexMessageType;
@ -82,8 +80,6 @@ public class FileService {
try {
hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getData()));
} catch (FeignException e) {
log.warn("Failed to optimize file: {}", request.getFilename(), e);
throw new BadRequestException("Failed to optimize document");

View File

@ -351,12 +351,14 @@ public class FileStatusService {
}
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate){
public void updateFileModificationDate(String fileId, OffsetDateTime fileManipulationDate) {
fileStatusPersistenceService.updateFileModificationDate(fileId, fileManipulationDate);
}
public void updateHasHighlights(String fileId, boolean hasHighlights){
public void updateHasHighlights(String fileId, boolean hasHighlights) {
fileStatusPersistenceService.updateHasHighlights(fileId, hasHighlights);
}

View File

@ -0,0 +1,21 @@
databaseChangeLog:
- changeSet:
id: add-index-information
author: philipp
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
primaryKeyName: index_information_pkey
name: id
type: VARCHAR(255)
- column:
name: index_configuration_hash
type: VARCHAR(255)
- column:
name: update_date
type: TIMESTAMP WITHOUT TIME ZONE
tableName: index_information

View File

@ -44,4 +44,6 @@ databaseChangeLog:
- include:
file: db/changelog/18-add-migration-table.yaml
- include:
file: db/changelog/19-added-has-highlights-to-file.changelog.yaml
file: db/changelog/19-added-has-highlights-to-file.changelog.yaml
- include:
file: db/changelog/20-add-index-information-table.yaml

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.peristence.v1.server.integration.client;
import org.springframework.cloud.openfeign.FeignClient;
import com.iqser.red.service.persistence.service.v1.api.resources.IndexInformationResource;
@FeignClient(name = "IndexInformationClient", url = "http://localhost:${server.port}")
public interface IndexInformationClient extends IndexInformationResource {
}

View File

@ -0,0 +1,62 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.peristence.v1.server.integration.client.IndexInformationClient;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository;
import com.iqser.red.service.persistence.service.v1.api.model.index.IndexInformation;
import feign.FeignException;
public class IndexInformationTest extends AbstractPersistenceServerServiceTest {
@Autowired
private IndexInformationClient indexInformationClient;
@Autowired
private IndexInformationRepository indexInformationRepository;
@Test
public void test() {
// No indexInformation exist
try {
IndexInformation indexInformation = indexInformationClient.getIndexInformation();
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(404);
}
// Save indexInformation the first time
String hash = "This is the hash from the search-service";
OffsetDateTime timeStamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
IndexInformation indexInformation = IndexInformation.builder().indexConfigurationHash(hash).updateDate(timeStamp).build();
indexInformationClient.updateIndexInformation(indexInformation);
IndexInformation indexInformationFromDb = indexInformationClient.getIndexInformation();
assertThat(indexInformationFromDb).isNotNull();
assertThat(indexInformationFromDb.getIndexConfigurationHash()).isEqualTo(hash);
assertThat(indexInformationFromDb.getUpdateDate()).isEqualTo(timeStamp);
// Update indexInformation
String newHash = "This is 2. the hash from the search-service";
OffsetDateTime newTimeStamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
IndexInformation newIndexInformation = IndexInformation.builder().indexConfigurationHash(newHash).updateDate(newTimeStamp).build();
indexInformationClient.updateIndexInformation(newIndexInformation);
indexInformationFromDb = indexInformationClient.getIndexInformation();
assertThat(indexInformationFromDb).isNotNull();
assertThat(indexInformationFromDb.getIndexConfigurationHash()).isEqualTo(newHash);
assertThat(indexInformationFromDb.getUpdateDate()).isEqualTo(newTimeStamp);
}
}

View File

@ -1,24 +1,13 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import com.iqser.red.service.pdftron.redaction.v1.api.model.*;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.when;
import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.amqp.core.AmqpAdmin;
@ -32,17 +21,60 @@ import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.when;
import java.util.List;
import java.util.TimeZone;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionResult;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.AuditRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DigitalSignatureRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierAttributeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ForceRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ImageRecategorizationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.IndexInformationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisChangeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.LegalBasisMappingRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationPreferencesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.NotificationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ReportTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.RuleSetRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.SMTPRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
@RunWith(SpringRunner.class)
@EnableFeignClients(basePackageClasses = FileClient.class)
@ -120,6 +152,9 @@ public abstract class AbstractPersistenceServerServiceTest {
protected MetricsPrinterService metricsPrinterService;
@Autowired
protected EntryRepository entryRepository;
@Autowired
protected IndexInformationRepository indexInformationRepository;
@Before
public void setupOptimize() {
@ -194,6 +229,7 @@ public abstract class AbstractPersistenceServerServiceTest {
dossierStatusRepository.deleteAll();
dossierTemplateRepository.deleteAll();
notificationPreferencesRepository.deleteAll();
indexInformationRepository.deleteAll();
}
static class Initializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {