Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Pull request #362: quartz jobs RED-3848

Browse Source 
Merge in RED/persistence-service from RED-3848 to master

* commit 'b47dabbc8f9545658e455bd806c726c1b6970199':
  RED-3800 liquibase revert
  RED-3800 test fix
  quartz jobs RED-3848
This commit is contained in:
Timo Bejan 2022-04-19 15:20:41 +02:00
commit c3fa21b92c
8 changed files with 641 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.dev/docker-compose.yaml
View File

@ -11,13 +11,12 @@ services:
POSTGRES_USER: redaction
POSTGRES_PASSWORD: redaction
POSTGRES_DB: redaction
rabbitmq:
image: 'bitnami/rabbitmq:latest'
mem_limit: 1000m
image: 'rabbitmq:3.9-alpine'
mem_limit: 500m
environment:
- RABBITMQ_PASSWORD=rabbitmq
- RABBITMQ_DEFAULT_USER=user
- RABBITMQ_DEFAULT_PASS=rabbitmq
ports:
- 5672:5672
- 15672:15672

10
persistence-service-v1/persistence-service-server-v1/pom.xml
View File

@ -139,24 +139,27 @@
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-quartz</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
<version>2.3.1.RELEASE</version>
</dependency>
<!-- test -->
<dependency>
<groupId>org.springframework.amqp</groupId>
<artifactId>spring-rabbit-test</artifactId>
<version>2.3.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<version>1.16.3</version>
<version>1.17.1</version>
<scope>test</scope>
</dependency>
<dependency>
@ -219,4 +222,5 @@
</plugin>
</plugins>
</build>
</project>

70
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/jobs/CreateJobsConfiguration.java Normal file
View File

@ -0,0 +1,70 @@
package com.iqser.red.service.peristence.v1.server.jobs;
import com.iqser.red.service.peristence.v1.server.service.job.AutomaticAnalysisJob;
import com.iqser.red.service.peristence.v1.server.service.job.DeletedFilesCleanupJob;
import com.iqser.red.service.peristence.v1.server.service.job.DownloadCleanupJob;
import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.text.ParseException;
@Configuration
public class CreateJobsConfiguration {
@Bean
public JobDetail automaticAnalysisJobDetail() {
return JobBuilder.newJob().ofType(AutomaticAnalysisJob.class)
.storeDurably()
.withIdentity("AutomaticAnalysisJob")
.withDescription("Automatically queues files for analysis")
.build();
}
@Bean
public Trigger automaticAnalysisJobTrigger() throws ParseException {
return TriggerBuilder.newTrigger().forJob(automaticAnalysisJobDetail())
.withIdentity("AutomaticAnalysisJobTrigger")
.withDescription("Triggers AutomaticAnalysisJob every 10 seconds")
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("*/10 * * * * ?")))
.build();
}
@Bean
public JobDetail deletedFilesCleanupJobDetail() {
return JobBuilder.newJob().ofType(DeletedFilesCleanupJob.class)
.storeDurably()
.withIdentity("DeletedFilesCleanupJob")
.withDescription("Hard delete dossiers / files after certain time")
.build();
}
@Bean
public Trigger deletedFilesCleanupJobTrigger() throws ParseException {
return TriggerBuilder.newTrigger().forJob(deletedFilesCleanupJobDetail())
.withIdentity("DeletedFilesCleanupJobTrigger")
.withDescription("Triggers DeletedFilesCleanupJob every 30 minutes")
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?")))
.build();
}
@Bean
public JobDetail downloadCleanupJobDetail() {
return JobBuilder.newJob().ofType(DownloadCleanupJob.class)
.storeDurably()
.withIdentity("DownloadCleanupJob")
.withDescription("Hard delete dossiers / files after certain time")
.build();
}
@Bean
public Trigger downloadCleanupJobTrigger() throws ParseException {
return TriggerBuilder.newTrigger().forJob(downloadCleanupJobDetail())
.withIdentity("DownloadCleanupJobTrigger")
.withDescription("Triggers DownloadCleanupJob every 30 minutes")
.withSchedule(CronScheduleBuilder.cronSchedule(new CronExpression("0 */30 * * * ?")))
.build();
}
}

20
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java → persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/AutomaticAnalysisJob.java
View File

@ -1,30 +1,25 @@
package com.iqser.red.service.peristence.v1.server.service.scheduler;
package com.iqser.red.service.peristence.v1.server.service.job;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.amqp.core.AmqpAdmin;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@RequiredArgsConstructor
public class AutomaticAnalysisScheduler {
public class AutomaticAnalysisJob implements Job {
@Value("${persistence-service.automaticAnalysis.pageFactor:500}")
private int pageFactor;
@ -35,16 +30,15 @@ public class AutomaticAnalysisScheduler {
@PostConstruct
protected void postConstruct() {
log.info("Automatic Analysis pageFactor: {}", pageFactor);
}
@Override
@Transactional
@Scheduled(fixedDelay = 10000, initialDelay = 10000)
public void checkFilesThatRequireReanalysisAndQueueIfPossible() {
public void execute(JobExecutionContext jobExecutionContext) {
if(fileManagementServiceSettings.isMigrateOnly()){
if (fileManagementServiceSettings.isMigrateOnly()) {
log.info("Skipping scheduling during migration");
return;
}

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java → persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DeletedFilesCleanupJob.java
View File

@ -1,9 +1,10 @@
package com.iqser.red.service.peristence.v1.server.service.scheduler;
package com.iqser.red.service.peristence.v1.server.service.job;
import java.time.OffsetDateTime;
import java.util.List;
import org.springframework.scheduling.annotation.Scheduled;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
@ -11,7 +12,6 @@ import com.iqser.red.service.peristence.v1.server.service.FileService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -19,7 +19,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Service
public class DeletedFilesCleanupService {
public class DeletedFilesCleanupJob implements Job {
private final DossierService dossierService;
private final FileStatusService fileStatusService;
@ -27,8 +27,8 @@ public class DeletedFilesCleanupService {
private final FileManagementServiceSettings settings;
@Scheduled(fixedDelay = 300000, initialDelay = 300000)
public void timedDeletion() {
@Override
public void execute(JobExecutionContext jobExecutionContext) {
var now = OffsetDateTime.now();
List<DossierEntity> dossiers = dossierService.getAllDossiers();
@ -53,4 +53,5 @@ public class DeletedFilesCleanupService {
}
}
}

14
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadCleanupService.java → persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/job/DownloadCleanupJob.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.peristence.v1.server.service.download;
package com.iqser.red.service.peristence.v1.server.service.job;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
@ -7,7 +7,8 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.springframework.stereotype.Service;
import java.time.OffsetDateTime;
@ -17,15 +18,15 @@ import java.util.List;
@Slf4j
@RequiredArgsConstructor
@Service
public class DownloadCleanupService {
public class DownloadCleanupJob implements Job {
private final DownloadStatusPersistenceService downloadStatusPersistenceService;
private final StorageService storageService;
private final FileManagementServiceSettings settings;
private final DossierService dossierService;
@Scheduled(fixedDelay = 300000, initialDelay = 300000)
public void processDocuments() {
@Override
public void execute(JobExecutionContext jobExecutionContext) {
var now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
log.info("Checking for downloads to delete at {}", now);
@ -56,7 +57,7 @@ public class DownloadCleanupService {
}
public void deleteDownload(DownloadStatusEntity downloadStatus) {
private void deleteDownload(DownloadStatusEntity downloadStatus) {
log.info("Deleting download status {}", downloadStatus);
try {
@ -70,4 +71,5 @@ public class DownloadCleanupService {
log.warn("DownloadStatus could not be deleted: {}" + e.getMessage());
}
}
}

536
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/23-quartz.changelog.yaml Normal file
View File

@ -0,0 +1,536 @@
databaseChangeLog:
- changeSet:
id: 1646818341589-1
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
name: BLOB_DATA
type: BLOB
tableName: QRTZ_BLOB_TRIGGERS
- changeSet:
id: 1646818341589-2
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: CALENDAR_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: CALENDAR
type: BLOB
tableName: QRTZ_CALENDARS
- changeSet:
id: 1646818341589-3
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: CRON_EXPRESSION
type: VARCHAR(200)
- column:
name: TIME_ZONE_ID
type: VARCHAR(80)
tableName: QRTZ_CRON_TRIGGERS
- changeSet:
id: 1646818341589-4
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: ENTRY_ID
type: VARCHAR(95)
- column:
constraints:
nullable: false
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: INSTANCE_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: FIRED_TIME
type: BIGINT
- column:
constraints:
nullable: false
name: SCHED_TIME
type: BIGINT
- column:
constraints:
nullable: false
name: PRIORITY
type: INT
- column:
constraints:
nullable: false
name: STATE
type: VARCHAR(16)
- column:
name: JOB_NAME
type: VARCHAR(200)
- column:
name: JOB_GROUP
type: VARCHAR(200)
- column:
name: IS_NONCONCURRENT
type: VARCHAR(1)
- column:
name: REQUESTS_RECOVERY
type: VARCHAR(1)
tableName: QRTZ_FIRED_TRIGGERS
- changeSet:
id: 1646818341589-5
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: JOB_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: JOB_GROUP
type: VARCHAR(200)
- column:
name: DESCRIPTION
type: VARCHAR(250)
- column:
constraints:
nullable: false
name: JOB_CLASS_NAME
type: VARCHAR(250)
- column:
constraints:
nullable: false
name: IS_DURABLE
type: VARCHAR(1)
- column:
constraints:
nullable: false
name: IS_NONCONCURRENT
type: VARCHAR(1)
- column:
constraints:
nullable: false
name: IS_UPDATE_DATA
type: VARCHAR(1)
- column:
constraints:
nullable: false
name: REQUESTS_RECOVERY
type: VARCHAR(1)
- column:
name: JOB_DATA
type: BLOB
tableName: QRTZ_JOB_DETAILS
- changeSet:
id: 1646818341589-6
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: LOCK_NAME
type: VARCHAR(40)
tableName: QRTZ_LOCKS
- changeSet:
id: 1646818341589-7
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
tableName: QRTZ_PAUSED_TRIGGER_GRPS
- changeSet:
id: 1646818341589-8
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: INSTANCE_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: LAST_CHECKIN_TIME
type: BIGINT
- column:
constraints:
nullable: false
name: CHECKIN_INTERVAL
type: BIGINT
tableName: QRTZ_SCHEDULER_STATE
- changeSet:
id: 1646818341589-9
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: REPEAT_COUNT
type: BIGINT
- column:
constraints:
nullable: false
name: REPEAT_INTERVAL
type: BIGINT
- column:
constraints:
nullable: false
name: TIMES_TRIGGERED
type: BIGINT
tableName: QRTZ_SIMPLE_TRIGGERS
- changeSet:
id: 1646818341589-10
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
name: STR_PROP_1
type: VARCHAR(512)
- column:
name: STR_PROP_2
type: VARCHAR(512)
- column:
name: STR_PROP_3
type: VARCHAR(512)
- column:
name: INT_PROP_1
type: INT
- column:
name: INT_PROP_2
type: INT
- column:
name: LONG_PROP_1
type: BIGINT
- column:
name: LONG_PROP_2
type: BIGINT
- column:
name: DEC_PROP_1
type: DECIMAL(13, 4)
- column:
name: DEC_PROP_2
type: DECIMAL(13, 4)
- column:
name: BOOL_PROP_1
type: VARCHAR(1)
- column:
name: BOOL_PROP_2
type: VARCHAR(1)
tableName: QRTZ_SIMPROP_TRIGGERS
- changeSet:
id: 1646818341589-11
author: timobejan (generated)
changes:
- createTable:
columns:
- column:
constraints:
nullable: false
primaryKey: true
name: SCHED_NAME
type: VARCHAR(120)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
primaryKey: true
name: TRIGGER_GROUP
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: JOB_NAME
type: VARCHAR(200)
- column:
constraints:
nullable: false
name: JOB_GROUP
type: VARCHAR(200)
- column:
name: DESCRIPTION
type: VARCHAR(250)
- column:
name: NEXT_FIRE_TIME
type: BIGINT
- column:
name: PREV_FIRE_TIME
type: BIGINT
- column:
name: PRIORITY
type: INT
- column:
constraints:
nullable: false
name: TRIGGER_STATE
type: VARCHAR(16)
- column:
constraints:
nullable: false
name: TRIGGER_TYPE
type: VARCHAR(8)
- column:
constraints:
nullable: false
name: START_TIME
type: BIGINT
- column:
name: END_TIME
type: BIGINT
- column:
name: CALENDAR_NAME
type: VARCHAR(200)
- column:
name: MISFIRE_INSTR
type: SMALLINT
- column:
name: JOB_DATA
type: BLOB
tableName: QRTZ_TRIGGERS
- changeSet:
id: 1646818341589-95
author: timobejan (generated)
changes:
- createIndex:
columns:
- column:
name: SCHED_NAME
- column:
name: JOB_NAME
- column:
name: JOB_GROUP
indexName: SCHED_NAME
tableName: QRTZ_TRIGGERS
- changeSet:
id: 1646818341589-218
author: timobejan (generated)
changes:
- addForeignKeyConstraint:
baseColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
baseTableName: QRTZ_BLOB_TRIGGERS
constraintName: QRTZ_BLOB_TRIGGERS_ibfk_1
deferrable: false
initiallyDeferred: false
onDelete: RESTRICT
onUpdate: RESTRICT
referencedColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
referencedTableName: QRTZ_TRIGGERS
validate: true
- changeSet:
id: 1646818341589-219
author: timobejan (generated)
changes:
- addForeignKeyConstraint:
baseColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
baseTableName: QRTZ_CRON_TRIGGERS
constraintName: QRTZ_CRON_TRIGGERS_ibfk_1
deferrable: false
initiallyDeferred: false
onDelete: RESTRICT
onUpdate: RESTRICT
referencedColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
referencedTableName: QRTZ_TRIGGERS
validate: true
- changeSet:
id: 1646818341589-220
author: timobejan (generated)
changes:
- addForeignKeyConstraint:
baseColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
baseTableName: QRTZ_SIMPLE_TRIGGERS
constraintName: QRTZ_SIMPLE_TRIGGERS_ibfk_1
deferrable: false
initiallyDeferred: false
onDelete: RESTRICT
onUpdate: RESTRICT
referencedColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
referencedTableName: QRTZ_TRIGGERS
validate: true
- changeSet:
id: 1646818341589-221
author: timobejan (generated)
changes:
- addForeignKeyConstraint:
baseColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
baseTableName: QRTZ_SIMPROP_TRIGGERS
constraintName: QRTZ_SIMPROP_TRIGGERS_ibfk_1
deferrable: false
initiallyDeferred: false
onDelete: RESTRICT
onUpdate: RESTRICT
referencedColumnNames: SCHED_NAME,TRIGGER_NAME,TRIGGER_GROUP
referencedTableName: QRTZ_TRIGGERS
validate: true
- changeSet:
id: 1646818341589-222
author: timobejan (generated)
changes:
- addForeignKeyConstraint:
baseColumnNames: SCHED_NAME,JOB_NAME,JOB_GROUP
baseTableName: QRTZ_TRIGGERS
constraintName: QRTZ_TRIGGERS_ibfk_1
deferrable: false
initiallyDeferred: false
onDelete: RESTRICT
onUpdate: RESTRICT
referencedColumnNames: SCHED_NAME,JOB_NAME,JOB_GROUP
referencedTableName: QRTZ_JOB_DETAILS
validate: true

2
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -53,3 +53,5 @@ databaseChangeLog:
file: db/changelog/sql/22-update-file-dossier-attributes.sql
- include:
file: db/changelog/22-add-soft-delete-time-to-entity.changelog.yaml
- include:
file: db/changelog/23-quartz.changelog.yaml