Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

performance updates

Browse Source
This commit is contained in:
Timo Bejan 2022-02-16 21:35:49 +02:00
parent 825566e81a
commit d34473f684
54 changed files with 771 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java
View File

@ -2,10 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.*;
import java.util.Map;
@ -13,6 +10,7 @@ import java.util.Map;
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(of ={ "dossierId"})
public class DossierStats {
private String dossierId;

4
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java
View File

@ -59,11 +59,13 @@ public class FileModel {
private Set<Integer> excludedPages = new HashSet<>();
private Map<String, String> fileAttributes = new HashMap<>();
private String dossierId;
private String dossierTemplateId;
private OffsetDateTime annotationModificationDate;
public boolean isAnalysisRequired(){
return this.fullAnalysisRequired || this.reanalysisRequired;
}
public boolean isSoftOrHardDeleted() {
return deleted != null || hardDeletedTime != null || ProcessingStatus.DELETED.equals(processingStatus);
}
}

5
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java
View File

@ -31,11 +31,6 @@ public interface StatusResource {
String USER_ID_REQUEST_PARAM = "userId";
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@GetMapping(value = STATUS_PATH, produces = MediaType.APPLICATION_JSON_VALUE)
List<FileModel> getAllStatuses();
@ResponseBody
@ResponseStatus(value = HttpStatus.OK)
@PostMapping(value = STATUS_PATH+DOSSIER_ID_PATH_PARAM+CHANGES_PATH, produces = MediaType.APPLICATION_JSON_VALUE)

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java
View File

@ -6,6 +6,9 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
@ -51,9 +54,10 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation {
private OffsetDateTime softDeletedTime;
@ElementCollection
@Fetch(FetchMode.SUBSELECT)
private List<RectangleEntity> positions = new ArrayList<>();
@ManyToOne
@ManyToOne(fetch = FetchType.LAZY)
private FileEntity fileStatus;
@Column

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java
View File

@ -6,6 +6,8 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
@ -38,10 +40,11 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation {
@Column(length = 4000)
private String value;
@ManyToOne
@ManyToOne(fetch = FetchType.LAZY)
private FileEntity fileStatus;
@ElementCollection
@Fetch(FetchMode.SUBSELECT)
private List<RectangleEntity> positions = new ArrayList<>();
@Column

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.audit;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -37,7 +38,7 @@ public class AuditEntity {
@Basic(fetch = FetchType.EAGER)
@Column(columnDefinition = "text")
@Convert(converter = JSONConverter.class)
@Convert(converter = JSONMapConverter.class)
private Map<String, Object> details = new HashMap<>();
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java
View File

@ -4,6 +4,8 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.util.ArrayList;
@ -23,6 +25,7 @@ public class LegalBasisMappingEntity {
private long version;
@ElementCollection
@Fetch(FetchMode.SUBSELECT)
private List<LegalBasisEntity> legalBasis = new ArrayList<>();
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java
View File

@ -7,6 +7,9 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.util.ArrayList;
@ -45,19 +48,19 @@ public class TypeEntity {
@Column
private boolean addToDictionaryAction;
@Fetch(FetchMode.SELECT)
@BatchSize(size=500)
@OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY)
private List<DictionaryEntryEntity> entries = new ArrayList<>();
@ManyToOne
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "dossier_template_id")
private DossierTemplateEntity dossierTemplate;
@Column(updatable = false, insertable = false, name = "dossier_template_id")
private String dossierTemplateId;
@ManyToOne
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "dossier_id")
private DossierEntity dossier;

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java
View File

@ -7,6 +7,9 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
@ -31,11 +34,11 @@ public class WatermarkEntity {
private int fontSize;
@Column
private String fontType;
@Column
@Enumerated(EnumType.STRING)
private WatermarkOrientation orientation;
@JsonIgnore
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "dossier_template_id")
private DossierTemplateEntity dossierTemplate;

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java
View File

@ -25,16 +25,6 @@ public class DossierAttributeEntity {
@Column(columnDefinition = "TEXT")
private String value;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@MapsId("dossierId")
private DossierEntity dossier;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@MapsId("dossierAttributeConfigId")
private DossierAttributeConfigEntity dossierAttributeConfig;
@Data
@NoArgsConstructor
@AllArgsConstructor

28
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java
View File

@ -2,12 +2,13 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.*;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
@ -22,6 +23,7 @@ import java.util.Set;
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "dossier")
@EqualsAndHashCode(of = {"id"})
public class DossierEntity {
@Id
@ -46,16 +48,18 @@ public class DossierEntity {
private String ownerId;
@Builder.Default
@ElementCollection
@Column(columnDefinition = "text", name = "members")
@Convert(converter = JSONStringSetConverter.class)
private Set<String> memberIds = new HashSet<>();
@Builder.Default
@ElementCollection
@Column(columnDefinition = "text", name = "approvers")
@Convert(converter = JSONStringSetConverter.class)
private Set<String> approverIds = new HashSet<>();
@Builder.Default
@ElementCollection
@Enumerated(EnumType.STRING)
@Column(columnDefinition = "text", name = "download_file_types")
@Convert(converter = JSONDownloadFileTypeConverter.class)
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
@Column
@ -87,15 +91,19 @@ public class DossierEntity {
private String dossierTemplateId;
@Builder.Default
@Fetch(FetchMode.SUBSELECT)
@ManyToMany(fetch = FetchType.LAZY, mappedBy = "dossiers")
private List<ReportTemplateEntity> reportTemplates = new ArrayList<>();
@Builder.Default
@OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier")
@Fetch(FetchMode.SUBSELECT)
@OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier", fetch = FetchType.LAZY)
private List<TypeEntity> dossierTypes = new ArrayList<>();
@Builder.Default
@OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier")
@Fetch(FetchMode.SUBSELECT)
@JoinColumn(name = "dossier_id")
@OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY)
private List<DossierAttributeEntity> dossierAttributes = new ArrayList<>();
@ManyToOne(fetch = FetchType.LAZY)

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java
View File

@ -4,6 +4,8 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import org.hibernate.annotations.Formula;
import javax.persistence.*;
@ -37,7 +39,8 @@ public class DossierStatusEntity {
@Column(updatable = false, insertable = false, name = "dossier_template_id")
private String dossierTemplateId;
@OneToMany(mappedBy = "dossierStatus")
@OneToMany(mappedBy = "dossierStatus", fetch = FetchType.LAZY)
@Fetch(FetchMode.SUBSELECT)
private List<DossierEntity> dossiers = new ArrayList<>();
}

39
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java
View File

@ -6,18 +6,11 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.ElementCollection;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Id;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Transient;
import javax.persistence.*;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
@ -25,6 +18,8 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
@Data
@Entity
@ -64,31 +59,37 @@ public class DossierTemplateEntity {
@Column(name = "soft_delete_time")
private OffsetDateTime softDeleteTime;
@ElementCollection
@Enumerated(EnumType.STRING)
@Builder.Default
@Column(columnDefinition = "text", name = "download_file_types")
@Convert(converter = JSONDownloadFileTypeConverter.class)
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
@JsonIgnore
@Builder.Default
@OneToMany(mappedBy = "dossierTemplate")
@Fetch(FetchMode.SUBSELECT)
private List<ReportTemplateEntity> reportTemplates = new ArrayList<>();
@JsonIgnore
@Builder.Default
@OneToMany(mappedBy = "dossierTemplate")
@Fetch(FetchMode.SUBSELECT)
private List<TypeEntity> dossierTypes = new ArrayList<>();
@JsonIgnore
@Builder.Default
@OneToMany(mappedBy = "dossierTemplate")
@Fetch(FetchMode.SUBSELECT)
private List<DossierEntity> dossiers = new ArrayList<>();
@JsonIgnore
@Builder.Default
@Fetch(FetchMode.SUBSELECT)
@OneToMany(mappedBy = "dossierTemplate")
private List<FileAttributeConfigEntity> fileAttributeConfigs = new ArrayList<>();
@Builder.Default
@Fetch(FetchMode.SUBSELECT)
@OneToMany(mappedBy = "dossierTemplate")
private List<DossierStatusEntity> dossierStatusList = new ArrayList<>();
@Transient
private DossierTemplateStatus dossierTemplateStatus;
@JsonIgnore
@OneToMany(mappedBy = "dossierTemplate")
private List<DossierStatusEntity> dossierStatusList = new ArrayList<>();
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java
View File

@ -1,6 +1,5 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import javax.persistence.*;
@ -17,17 +16,6 @@ public class FileAttributeEntity {
@Column(length = 4000)
private String value;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@MapsId("fileId")
private FileEntity file;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
@MapsId("fileAttributeConfigId")
private FileAttributeConfigEntity fileAttributeConfig;
@Data
@Embeddable
public static class FileAttributeEntityId implements Serializable {

17
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java
View File

@ -1,12 +1,14 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.dossier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
@ -133,17 +135,16 @@ public class FileEntity {
@Column
private OffsetDateTime annotationModificationDate;
@ElementCollection(fetch = FetchType.EAGER)
@Column(columnDefinition = "text", name = "excluded_pages")
@Convert(converter = JSONIntegerSetConverter.class)
private Set<Integer> excludedPages = new HashSet<>();
@OneToMany(mappedBy = "file", fetch = FetchType.EAGER, cascade = CascadeType.ALL)
@JoinColumn(name = "file_id")
@Fetch(FetchMode.SUBSELECT)
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL)
private List<FileAttributeEntity> fileAttributes = new ArrayList<>();
@JsonIgnore
@ManyToOne
private DossierEntity dossier;
@Column(updatable = false, insertable = false, name = "dossier_id")
@Column(name = "dossier_id")
private String dossierId;
public boolean isSoftDeleted() {

16
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java
View File

@ -2,17 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.downloa
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.BatchSize;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Data
@Entity
@ -40,15 +46,17 @@ public class DownloadStatusEntity {
@Column
private long fileSize;
@ManyToOne
@ManyToOne(fetch = FetchType.LAZY)
private DossierEntity dossier;
@ManyToMany
@Fetch(FetchMode.SUBSELECT)
private List<FileEntity> files = new ArrayList<>();
@ElementCollection
@Enumerated(EnumType.STRING)
private List<DownloadFileType> downloadFileTypes = new ArrayList<>();
@Builder.Default
@Column(columnDefinition = "text", name = "download_file_types")
@Convert(converter = JSONDownloadFileTypeConverter.class)
private Set<DownloadFileType> downloadFileTypes = new HashSet<>();
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.entity.notification;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -49,6 +50,6 @@ public class NotificationEntity {
@Basic(fetch = FetchType.EAGER)
@Column(columnDefinition = "text")
@Convert(converter = JSONConverter.class)
@Convert(converter = JSONMapConverter.class)
private Map<String, Object> target = new HashMap<>();
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java
View File

@ -5,6 +5,8 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import javax.persistence.*;
import java.util.ArrayList;
@ -31,8 +33,10 @@ public class NotificationPreferencesEntity {
private EmailNotificationType emailNotificationType;
@ElementCollection
@Fetch(FetchMode.SUBSELECT)
private List<String> emailNotifications = new ArrayList<>();
@ElementCollection
@Fetch(FetchMode.SUBSELECT)
private List<String> inAppNotifications = new ArrayList<>();
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java
View File

@ -24,11 +24,9 @@ public class DossierAttributePersistenceService {
public void insertDossierAttribute(String dossierId, String dossierAttributeId, String dossierAttributeValue) {
DossierAttributeEntity dossierAttribute = new DossierAttributeEntity();
dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(null, null));
dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(dossierId, dossierAttributeId));
dossierAttribute.setValue(dossierAttributeValue);
dossierAttribute.setDossier(dossierRepository.getOne(dossierId));
dossierAttribute.setDossierAttributeConfig(dossierAttributeConfigRepository.getOne(dossierAttributeId));
dossierAttributeRepository.save(dossierAttribute);
dossierAttributeRepository.saveAndFlush(dossierAttribute);
}
@Transactional

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java
View File

@ -14,6 +14,7 @@ import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@ -36,7 +37,7 @@ public class DownloadStatusPersistenceService {
downloadStatus.setDossier(dossier);
downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
downloadStatus.setFiles(fileRepository.findAllById(fileIds));
downloadStatus.setDownloadFileTypes(new ArrayList<>(downloadFileTypes));
downloadStatus.setDownloadFileTypes(new HashSet<>(downloadFileTypes));
downloadStatusRepository.save(downloadStatus);
}

34
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java
View File

@ -37,7 +37,7 @@ public class FileStatusPersistenceService {
FileEntity file = new FileEntity();
file.setId(fileId);
file.setDossier(dossierRepository.getOne(dossierId));
file.setDossierId(dossierId);
file.setFilename(filename);
file.setProcessingStatus(ProcessingStatus.UNPROCESSED);
file.setWorkflowStatus(WorkflowStatus.NEW);
@ -58,8 +58,9 @@ public class FileStatusPersistenceService {
if (isFileDeleted(fileId)) {
return;
}
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion,
legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now()
.truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber);
}
@ -167,9 +168,9 @@ public class FileStatusPersistenceService {
}
var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes);
file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
file.setFileAttributes(fileAttributeEntities);
fileAttributesRepository.saveAllAndFlush(fileAttributeEntities);
fileRepository.updateLastAttributeChangeDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
}, () -> {
throw new NotFoundException("Unknown file=" + fileId);
});
@ -178,8 +179,7 @@ public class FileStatusPersistenceService {
@Transactional
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date,
boolean hasSuggestions) {
public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, boolean hasSuggestions) {
if (isFileDeleted(fileId)) {
return;
@ -220,12 +220,6 @@ public class FileStatusPersistenceService {
}
public List<FileEntity> getAllStatuses() {
return fileRepository.findAll();
}
public List<FileEntity> getActiveFiles(String dossierId) {
return fileRepository.findByDossierId(dossierId)
@ -339,12 +333,13 @@ public class FileStatusPersistenceService {
return fileAttributesMap.entrySet().stream().map(entry -> {
var fa = new FileAttributeEntity();
fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId());
fa.setFile(file);
fa.setFileAttributeConfig(configuration.stream()
var id = new FileAttributeEntity.FileAttributeEntityId();
id.setFileId(file.getId());
id.setFileAttributeConfigId(configuration.stream()
.filter(c -> c.getId().equals(entry.getKey()))
.findAny()
.findAny().map(FileAttributeConfigEntity::getId)
.orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
fa.setFileAttributeId(id);
fa.setValue(entry.getValue());
return fa;
}).collect(Collectors.toList());
@ -369,4 +364,7 @@ public class FileStatusPersistenceService {
return fileRepository.countSoftDeletedFiles(dossierId);
}
public List<FileEntity> getAllRelevantStatusesForReanalysisScheduler() {
return fileRepository.getAllRelevantStatusesForReanalysisScheduler();
}
}

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java
View File

@ -11,12 +11,11 @@ public interface DossierAttributeRepository extends JpaRepository<DossierAttribu
List<DossierAttributeEntity> findByIdDossierId(String dossierId);
@Modifying(clearAutomatically = true, flushAutomatically = true)
@Query("DELETE FROM DossierAttributeEntity e WHERE e.id.dossierId = :id")
void deleteByDossierId(String dossierId);
@Query("SELECT a FROM DossierAttributeEntity a WHERE a.dossierAttributeConfig.dossierTemplate.id = :dossierTemplateId")
List<DossierAttributeEntity> findByDossierTemplateId(String dossierTemplateId);
@Modifying
@Modifying(clearAutomatically = true, flushAutomatically = true)
@Query("DELETE FROM DossierAttributeEntity e WHERE e.id.dossierAttributeConfigId = :id")
void deleteByDossierAttributeConfigId(String id);

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java
View File

@ -5,13 +5,18 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import java.util.List;
public interface FileAttributesRepository extends JpaRepository<FileAttributeEntity, String> {
@Modifying
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Query("DELETE FROM FileAttributeEntity f where f.fileAttributeId.fileAttributeConfigId = :fileAttributeConfigId")
void deleteByFileAttributeConfigId(String fileAttributeConfigId);
@Modifying
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Query("DELETE FROM FileAttributeEntity f where f.fileAttributeId.fileId = :fileId")
void deleteByFileId(String fileId);
@Query("SELECT f FROM FileAttributeEntity f where f.fileAttributeId.fileId = :fileId")
List<FileAttributeEntity> findByFileId(String fileId);
}

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java
View File

@ -139,6 +139,15 @@ public interface FileRepository extends JpaRepository<FileEntity, String> {
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.annotationModificationDate = :annotationModificationDate where f.id = :fileId")
void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime annotationModificationDate);
@Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " +
"and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'DELETED' or f.processingStatus = 'ERROR' )" +
"and d.softDeletedTime is null and d.hardDeletedTime is null and d.status = 'ACTIVE' " )
List<FileEntity> getAllRelevantStatusesForReanalysisScheduler();
@Modifying(clearAutomatically = true)
@Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date")
void updateLastAttributeChangeDate(OffsetDateTime date);
}

33
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONDownloadFileTypeConverter.java Normal file
View File

@ -0,0 +1,33 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import com.amazonaws.services.s3.transfer.Download;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import lombok.SneakyThrows;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import java.util.HashSet;
import java.util.Set;
@Converter
public class JSONDownloadFileTypeConverter implements AttributeConverter<Set<DownloadFileType>, String> {
private final ObjectMapper objectMapper = new ObjectMapper();
@SneakyThrows
@Override
public String convertToDatabaseColumn(Set<DownloadFileType> dataSet) {
return objectMapper.writeValueAsString(dataSet);
}
@SneakyThrows
@Override
public Set<DownloadFileType> convertToEntityAttribute(String data) {
TypeReference<HashSet<DownloadFileType>> typeRef = new TypeReference<>() {
};
return objectMapper.readValue(data, typeRef);
}
}

31
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONIntegerSetConverter.java Normal file
View File

@ -0,0 +1,31 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import java.util.HashSet;
import java.util.Set;
@Converter
public class JSONIntegerSetConverter implements AttributeConverter<Set<Integer>, String> {
private final ObjectMapper objectMapper = new ObjectMapper();
@SneakyThrows
@Override
public String convertToDatabaseColumn(Set<Integer> dataSet) {
return objectMapper.writeValueAsString(dataSet);
}
@SneakyThrows
@Override
public Set<Integer> convertToEntityAttribute(String data) {
TypeReference<HashSet<Integer>> typeRef = new TypeReference<>() {
};
return objectMapper.readValue(data, typeRef);
}
}

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONConverter.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONMapConverter.java
View File

@ -10,11 +10,10 @@ import java.util.HashMap;
import java.util.Map;
@Converter
public class JSONConverter implements AttributeConverter<Map<String, Object>, String> {
public class JSONMapConverter implements AttributeConverter<Map<String, Object>, String> {
private final ObjectMapper objectMapper = new ObjectMapper();
@SneakyThrows
@Override
public String convertToDatabaseColumn(Map<String, Object> map) {

31
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStringSetConverter.java Normal file
View File

@ -0,0 +1,31 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import javax.persistence.AttributeConverter;
import javax.persistence.Converter;
import java.util.HashSet;
import java.util.Set;
@Converter
public class JSONStringSetConverter implements AttributeConverter<Set<String>, String> {
private final ObjectMapper objectMapper = new ObjectMapper();
@SneakyThrows
@Override
public String convertToDatabaseColumn(Set<String> dataSet) {
return objectMapper.writeValueAsString(dataSet);
}
@SneakyThrows
@Override
public Set<String> convertToEntityAttribute(String data) {
TypeReference<HashSet<String>> typeRef = new TypeReference<>() {
};
return objectMapper.readValue(data, typeRef);
}
}

7
persistence-service-v1/persistence-service-server-v1/pom.xml
View File

@ -18,6 +18,7 @@
</properties>
<dependencies>
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>search-service-api-v1</artifactId>
@ -153,6 +154,12 @@
<artifactId>liquibase-core</artifactId>
<version>4.3.1</version>
</dependency>
<dependency>
<groupId>com.yannbriancon</groupId>
<artifactId>spring-hibernate-query-utils</artifactId>
<version>2.0.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java
View File

@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource;
import com.iqser.red.service.search.v1.model.IndexMessageType;
import feign.Param;
@ -74,7 +75,7 @@ public class DossierController implements DossierResource {
throw new DossierNotFoundException(String.format(DOSSIER_NOT_FOUND_MESSAGE, dossierId));
}
List<FileEntity> fileStatuses = fileStatusService.getDossierStatus(dossierId);
List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossierId);
fileStatuses.stream().filter(fileStatus -> fileStatus.getDeleted() == null).forEach(fileStatus -> {
fileService.softDeleteFile(dossierId, fileStatus.getId(), now);
fileStatusService.setFileStatusDeleted(fileStatus.getId(), now);
@ -171,7 +172,7 @@ public class DossierController implements DossierResource {
for (String dossierId : dossierIds) {
DossierEntity dossier = dossierService.getDossierById(dossierId);
dossierService.hardDeleteDossier(dossier.getId());
List<FileEntity> fileStatuses = fileStatusService.getDossierStatus(dossierId);
List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossierId);
fileStatuses.forEach(fileStatus -> {
fileService.hardDeleteFile(dossierId, fileStatus.getId());
fileStatusService.setFileStatusHardDeleted(fileStatus.getId());
@ -186,7 +187,7 @@ public class DossierController implements DossierResource {
for (String dossierId : dossierIds) {
var dossier = dossierService.getDossierById(dossierId);
List<FileEntity> fileStatuses = fileStatusService.getDossierStatus(dossierId);
List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossierId);
fileStatuses.forEach(fileStatus -> {
if (fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted()
.isAfter(dossier.getSoftDeletedTime()))) {
@ -208,7 +209,7 @@ public class DossierController implements DossierResource {
DossierEntity dossier = dossierService.getDossierById(dossierId);
if (dossier.getHardDeletedTime() == null) {
dossierService.archiveDossier(dossier.getId());
List<FileEntity> fileStatuses = fileStatusService.getDossierStatus(dossier.getId());
List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossier.getId());
addToIndexingQueue(dossierId, fileStatuses);
} else {
log.error("Could not archive Dossier {}, because it is already deleted.", dossierId);
@ -225,7 +226,7 @@ public class DossierController implements DossierResource {
DossierEntity dossier = dossierService.getDossierById(dossierId);
if (dossier.getHardDeletedTime() == null) {
dossierService.unarchiveDossier(dossier.getId());
List<FileEntity> fileStatuses = fileStatusService.getDossierStatus(dossier.getId());
List<FileModel> fileStatuses = fileStatusService.getDossierStatus(dossier.getId());
addToIndexingQueue(dossierId, fileStatuses);
} else {
log.error("Could not unarchive Dossier {}, because it is deleted.", dossierId);
@ -235,7 +236,7 @@ public class DossierController implements DossierResource {
}
private void addToIndexingQueue(String dossierId, List<FileEntity> fileStatuses) {
private void addToIndexingQueue(String dossierId, List<FileModel> fileStatuses) {
fileStatuses.forEach(f -> indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, f.getId(), 2));
}

5
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java
View File

@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvResponse;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.resources.FileAttributesResource;
import com.iqser.red.service.persistence.service.v1.api.utils.SuppressFBWarnings;
@ -60,10 +61,10 @@ public class FileAttributesController implements FileAttributesResource {
FileAttributesGeneralConfigurationEntity generalConfiguration = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossier.getDossierTemplateId());
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId());
Map<String, FileEntity> fileStatusByFilename = fileStatusService.getDossierStatus(dossierId)
Map<String, FileModel> fileStatusByFilename = fileStatusService.getDossierStatus(dossierId)
.stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.collect(Collectors.toMap(FileEntity::getFilename, Function.identity()));
.collect(Collectors.toMap(FileModel::getFilename, Function.identity()));
List<List<String>> rows = getCsvRecords(importCsvRequest.getCsvFile(), generalConfiguration.getDelimiter());

42
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java
View File

@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.service.v1.api.resources.StatusResource
import com.iqser.red.service.search.v1.model.IndexMessageType;
import lombok.RequiredArgsConstructor;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
@ -31,18 +32,8 @@ public class FileStatusController implements StatusResource {
private final FileStatusService fileStatusService;
private final ExcludeFromAnalysisService excludeFromAnalysis;
private final AnalysisFlagsCalculationService analysisFlagsCalculationService;
private final ReanalysisRequiredStatusService reanalysisRequiredStatusService;
private final IndexingService indexingService;
@Override
public List<FileModel> getAllStatuses() {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses()
.stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()));
}
@Override
public JSONPrimitive<Boolean> hasChangesSince(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@ -55,31 +46,23 @@ public class FileStatusController implements StatusResource {
@Override
public List<FileModel> getDossierStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId) {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId)
.stream()
return fileStatusService.getDossierStatus(dossierId).stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()));
.collect(Collectors.toList());
}
@Override
public List<FileModel> getSoftDeletedDossierStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId) {
var softDeletedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId)
.stream()
.filter(f -> f.getProcessingStatus().equals(ProcessingStatus.DELETED) && f.getHardDeletedTime() == null)
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()));
softDeletedFiles.sort((f1, f2) -> f2.getDeleted().compareTo(f1.getDeleted()));
return softDeletedFiles;
return fileStatusService.getDossierStatus(dossierId).stream()
.filter(f3 -> f3.getProcessingStatus().equals(ProcessingStatus.DELETED) && f3.getHardDeletedTime() == null)
.sorted((f11, f21) -> f21.getDeleted().compareTo(f11.getDeleted())).collect(Collectors.toList());
}
@Override
public FileModel getFileStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId,
@PathVariable(FILE_ID) String fileId) {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(
convert(fileStatusService.getStatus(fileId), FileModel.class, new FileModelMapper()));
return fileStatusService.getStatus(fileId);
}
@ -97,7 +80,7 @@ public class FileStatusController implements StatusResource {
@RequestParam(value = USER_ID_REQUEST_PARAM, required = false) String userId) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
String assignee = fileStatus.getAssignee();
if (userId != null) {
assignee = userId;
@ -112,7 +95,7 @@ public class FileStatusController implements StatusResource {
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = APPROVER_ID_REQUEST_PARAM, required = false) String approverId) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
String assignee = fileStatus.getAssignee();
if (approverId != null) {
assignee = approverId;
@ -128,7 +111,7 @@ public class FileStatusController implements StatusResource {
@PathVariable(FILE_ID) String fileId,
@RequestParam(value = APPROVER_ID_REQUEST_PARAM, required = false) String approverId) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
String assignee = fileStatus.getAssignee();
if (assignee == null && approverId == null) {
throw new BadRequestException("No user assigned for the approval");
@ -158,7 +141,8 @@ public class FileStatusController implements StatusResource {
public void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<Integer> pages) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
Set<Integer> excludedPages = fileStatus.getExcludedPages();
excludedPages.addAll(pages);
@ -169,7 +153,7 @@ public class FileStatusController implements StatusResource {
public void includePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set<Integer> pages) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
Set<Integer> excludedPages = fileStatus.getExcludedPages();
excludedPages.removeAll(pages);
fileStatusService.setExcludedPages(fileId, excludedPages);

15
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -95,7 +95,7 @@ public class ReanalysisController implements ReanalysisResource {
if (force) {
fileStatusService.setStatusOcrProcessing(dossierId, fileId);
} else {
FileEntity dossierFile = fileStatusService.getStatus(fileId);
FileModel dossierFile = fileStatusService.getStatus(fileId);
if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
throw new ConflictException("Cannot analyse a deleted/approved file");
}
@ -107,7 +107,7 @@ public class ReanalysisController implements ReanalysisResource {
}
private List<FileEntity> getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) {
private List<FileModel> getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -124,11 +124,11 @@ public class ReanalysisController implements ReanalysisResource {
}
private List<FileEntity> getRelevantFiles(String dossierId, Collection<String> fileIds) {
private List<FileModel> getRelevantFiles(String dossierId, Collection<String> fileIds) {
var dossierFiles = getAllFilesForDossier(dossierId, false);
var relevantFiles = new ArrayList<FileEntity>();
var relevantFiles = new ArrayList<FileModel>();
for (var fileId : fileIds) {
var dossierFileOptional = dossierFiles.stream().filter(f -> f.getId().equals(fileId)).findAny();
if (dossierFileOptional.isEmpty()) {
@ -150,17 +150,14 @@ public class ReanalysisController implements ReanalysisResource {
}
private void reanalyseFiles(String dossierId, boolean force, List<FileEntity> filesToReanalyse) {
private void reanalyseFiles(String dossierId, boolean force, List<FileModel> filesToReanalyse) {
if (force) {
filesToReanalyse.forEach(file -> {
fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true);
});
} else {
var enhancedAndConvertedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(
fileStatusService.getAllStatuses(), FileModel.class, new FileModelMapper()));
enhancedAndConvertedFiles.forEach(file -> {
filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> {
fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true);
});
}

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimiti
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
@ -91,7 +92,7 @@ public class UploadController implements UploadResource {
for (String fileId : fileIds) {
FileEntity fileStatus = fileStatusService.getStatus(fileId);
FileModel fileStatus = fileStatusService.getStatus(fileId);
OffsetDateTime softDeletedTime = fileStatus.getDeleted();
fileService.undeleteFile(dossier.getDossierTemplateId(), dossierId, fileId, softDeletedTime);

30
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java
View File

@ -178,6 +178,7 @@ public class MigrationService {
private void migrateRedactionLogs() {
List<FileEntity> files = fileRepository.findAll();
List<DossierEntity> dossiers = dossierRepository.findAll();
files.stream().filter(file -> file.getHardDeletedTime() == null).forEach(file -> {
migrateRedactionLog(file);
});
@ -341,7 +342,7 @@ public class MigrationService {
downloadStatus.setDossier(dossierRepository.getOne(download.getDossierId()));
downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS));
downloadStatus.setFiles(fileRepository.findAllById(download.getDownloadDetails().getFileIds()));
downloadStatus.setDownloadFileTypes(new ArrayList<>(download.getDownloadDetails().getDownloadFileTypes()));
downloadStatus.setDownloadFileTypes(new HashSet<>(download.getDownloadDetails().getDownloadFileTypes()));
downloadStatus.setStatus(download.getStatus());
downloadStatus.setFileSize(download.getFileSize());
downloadStatus.setLastDownload(download.getLastDownload());
@ -458,12 +459,13 @@ public class MigrationService {
imageRedactions.forEach(redaction -> {
var file = fileRepository.getById(redaction.getFileId());
var dossier = dossierRepository.getById(file.getDossierId());
var manualImageRecategorization = ManualImageRecategorizationEntity.builder()
.id(new AnnotationEntityId(redaction.getId(), redaction.getFileId()))
.user(redaction.getUser())
.status(redaction.getStatus())
.typeId(toTypeId(redaction.getType(), file.getDossier().getDossierTemplateId(), null))
.typeId(toTypeId(redaction.getType(), dossier.getDossierTemplateId(), null))
.requestDate(redaction.getRequestDate())
.processedDate(redaction.getProcessedDate())
.softDeletedTime(redaction.getSoftDeletedTime())
@ -569,14 +571,13 @@ public class MigrationService {
addRedactions.forEach(addRedaction -> {
var file = fileRepository.getById(addRedaction.getFileId());
var dossier = dossierRepository.findById(file.getDossierId());
ManualRedactionEntryEntity manualRedactionEntry = ManualRedactionEntryEntity.builder()
.id(new AnnotationEntityId(addRedaction.getId(), addRedaction.getFileId()))
.user(addRedaction.getUser())
.typeId(toTypeId(addRedaction.getType(), file.getDossier()
.getDossierTemplateId(), addRedaction.getType()
.equals("dossier_redaction") ? file.getDossier().getId() : null))
.typeId(toTypeId(addRedaction.getType(), dossier.get().getDossierTemplateId(), addRedaction.getType()
.equals("dossier_redaction") ? dossier.get().getDossierTemplateId() : null))
.value(addRedaction.getValue())
.reason(addRedaction.getReason())
.legalBasis(addRedaction.getLegalBasis())
@ -800,14 +801,12 @@ public class MigrationService {
.lastFileAttributeChange(oldFile.getLastFileAttributeChange())
.analysisVersion(oldFile.getAnalysisVersion())
.excludedPages(oldFile.getExcludedPages())
.dossier(dossierEntity)
.dossierId(dossierEntity.getId())
.dossierId(oldFile.getDossierId())
.build();
List<FileAttributeEntity> convertedFileAttributes = new ArrayList<>();
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(file
.getDossier()
.getDossierTemplateId());
List<FileAttributeConfigEntity> configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossierEntity.getDossierTemplateId());
if (oldFile.getFileAttributes() != null) {
oldFile.getFileAttributes().getAttributeIdToValue().entrySet().forEach(entry -> {
@ -816,16 +815,16 @@ public class MigrationService {
.stream()
.filter(nId -> nId.getDossierTemplateId().equals(dossierEntity.getDossierTemplateId()))
.findFirst();
if (!newId.isPresent()) {
if (newId.isEmpty()) {
return;
}
var fa = new FileAttributeEntity();
fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId());
fa.setFile(file);
fa.setFileAttributeConfig(configuration.stream()
fa.getFileAttributeId().setFileId(file.getId());
fa.getFileAttributeId().setFileAttributeConfigId(configuration.stream()
.filter(c -> c.getId().equals(newId.get().getNewId()))
.findAny()
.findAny().map(FileAttributeConfigEntity::getId)
.orElseThrow(() -> new BadRequestException("Invalid File Attribute Id")));
fa.setValue(entry.getValue());
convertedFileAttributes.add(fa);
@ -860,9 +859,6 @@ public class MigrationService {
dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(dossier.getId(), newConfigId.get()
.getNewId()));
dossierAttribute.setValue(oldDossierAttribute.getValue());
dossierAttribute.setDossier(dossier);
dossierAttribute.setDossierAttributeConfig(dossierAttributeConfigRepository.getOne(newConfigId.get()
.getNewId()));
dossierAttributeRepository.save(dossierAttribute);
}
});

24
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java
View File

@ -5,6 +5,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
@ -42,7 +43,12 @@ public class DossierStatsService {
}
public DossierStats getDossierStats(String dossierId) {
return dossierStatsMap.getOrDefault(dossierId, computeDossierStats(dossierService.getDossierById(dossierId)));
var stats = dossierStatsMap.get(dossierId);
if (stats == null) {
stats = computeDossierStats(dossierService.getDossierById(dossierId));
dossierStatsMap.put(dossierId, stats);
}
return stats;
}
private DossierStats computeDossierStats(DossierEntity dossierEntity) {
@ -56,21 +62,21 @@ public class DossierStatsService {
dossierStats.setDossierId(dossierId);
// get the associated files
List<FileEntity> files = fileStatusService.getActiveFiles(dossierId);
List<FileModel> files = fileStatusService.getActiveFiles(dossierId);
dossierStats.setNumberOfFiles(files.size());
dossierStats.setNumberOfSoftDeletedFiles(fileStatusService.countSoftDeletedFiles(dossierId));
dossierStats.setNumberOfPages(files.stream().mapToInt(FileEntity::getNumberOfPages).sum());
dossierStats.setNumberOfPages(files.stream().mapToInt(FileModel::getNumberOfPages).sum());
dossierStats.setNumberOfExcludedPages(files.stream().mapToInt(f -> f.getExcludedPages().size()).sum());
files.stream().filter(FileEntity::isHasRedactions).findAny().ifPresent(
files.stream().filter(FileModel::isHasRedactions).findAny().ifPresent(
(v) -> dossierStats.setHasRedactionsFilePresent(true)
);
files.stream().filter(FileEntity::isHasHints).filter(f -> !f.isHasRedactions()).findAny().ifPresent(
files.stream().filter(FileModel::isHasHints).filter(f -> !f.isHasRedactions()).findAny().ifPresent(
(v) -> dossierStats.setHasHintsNoRedactionsFilePresent(true)
);
files.stream().filter(FileEntity::isHasSuggestions).findAny().ifPresent(
files.stream().filter(FileModel::isHasSuggestions).findAny().ifPresent(
(v) -> dossierStats.setHasSuggestionsFilePresent(true)
);
files.stream().filter(FileEntity::isHasUpdates).findAny().ifPresent(
files.stream().filter(FileModel::isHasUpdates).findAny().ifPresent(
(v) -> dossierStats.setHasUpdatesFilePresent(true)
);
files.stream().filter(f -> !f.isHasRedactions())
@ -80,10 +86,10 @@ public class DossierStatsService {
.findAny().ifPresent(
(v) -> dossierStats.setHasNoFlagsFilePresent(true)
);
var fileCountPerProcessingStatus = files.stream().collect(Collectors.toMap(FileEntity::getProcessingStatus, e -> 1, Math::addExact));
var fileCountPerProcessingStatus = files.stream().collect(Collectors.toMap(FileModel::getProcessingStatus, e -> 1, Math::addExact));
dossierStats.setFileCountPerProcessingStatus(fileCountPerProcessingStatus);
var fileCountPerWorkflowStatus = files.stream().collect(Collectors.toMap(FileEntity::getWorkflowStatus, e -> 1, Math::addExact));
var fileCountPerWorkflowStatus = files.stream().collect(Collectors.toMap(FileModel::getWorkflowStatus, e -> 1, Math::addExact));
dossierStats.setFileCountPerWorkflowStatus(fileCountPerWorkflowStatus);
return dossierStats;
}

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java
View File

@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Arrays;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import org.springframework.stereotype.Service;
import com.google.common.hash.HashFunction;
@ -97,7 +98,7 @@ public class FileService {
}
private FileEntity retrieveStatus(String fileId) {
private FileModel retrieveStatus(String fileId) {
try {
return fileStatusService.getStatus(fileId);

107
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -1,16 +1,5 @@
package com.iqser.red.service.peristence.v1.server.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
@ -20,28 +9,37 @@ import com.iqser.red.service.peristence.v1.server.controller.RulesController;
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException;
import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.MessageType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@ -62,24 +60,43 @@ public class FileStatusService {
private final RemoveRedactionPersistenceService removeRedactionPersistenceService;
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final ResizeRedactionPersistenceService resizeRedactionPersistenceService;
private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService;
private final FileManagementServiceSettings settings;
private final ReanalysisRequiredStatusService reanalysisRequiredStatusService;
@Transactional
public List<FileModel> getAllRelevantStatusesForReanalysisScheduler() {
var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler();
var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList)
.stream()
.filter(FileModel::isAnalysisRequired)
.collect(Collectors.toList());
}
public List<FileEntity> getActiveFiles(String dossierId) {
@Transactional
public List<FileModel> getActiveFiles(String dossierId) {
return fileStatusPersistenceService.getActiveFiles(dossierId);
var fileEntities = fileStatusPersistenceService.getActiveFiles(dossierId);
var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
}
public List<FileEntity> getDossierStatus(String dossierId) {
@Transactional
public List<FileModel> getDossierStatus(String dossierId) {
return fileStatusPersistenceService.getStatusesForDossier(dossierId);
var fileEntities = new ArrayList<>(fileStatusPersistenceService.getStatusesForDossier(dossierId));
var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList);
}
public FileEntity getStatus(String fileId) {
return fileStatusPersistenceService.getStatus(fileId);
@Transactional
public FileModel getStatus(String fileId) {
var fileEntity = fileStatusPersistenceService.getStatus(fileId);
var converted = convert(fileEntity, FileModel.class, new FileModelMapper());
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(converted);
}
@ -169,7 +186,6 @@ public class FileStatusService {
@Transactional
@SuppressWarnings("PMD")
protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set<Integer> sectionsToReanalyse) {
var dossier = dossierPersistenceService.getAndValidateDossier(dossierId);
@ -205,7 +221,7 @@ public class FileStatusService {
.manualRedactions(manualRedactionProviderService.getManualRedactions(fileId))
.dossierTemplateId(dossier.getDossierTemplateId())
.lastProcessed(fileStatus.getLastProcessed())
.fileAttributes(convert(fileStatus.getFileAttributes()))
.fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId()))
.excludedPages(fileStatus.getExcludedPages())
.build();
@ -341,6 +357,12 @@ public class FileStatusService {
}
public void setStatusNerAnalyzing(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING);
}
public void setStatusOcrProcessing(String dossierId, String fileId) {
FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId);
@ -355,12 +377,6 @@ public class FileStatusService {
}
public void setStatusNerAnalyzing(String fileId) {
fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING);
}
private void addToImageQueue(String dossierId, String fileId) {
setStatusImageAnalyzing(fileId);
@ -403,12 +419,6 @@ public class FileStatusService {
}
public List<FileEntity> getAllStatuses() {
return fileStatusPersistenceService.getAllStatuses();
}
public void wipeFileData(String dossierId, String fileId) {
OffsetDateTime now = OffsetDateTime.now();
@ -454,16 +464,23 @@ public class FileStatusService {
}
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convert(
List<FileAttributeEntity> fileAttributes) {
private List<com.iqser.red.service.redaction.v1.model.FileAttribute> convertAttributes(
List<FileAttributeEntity> fileAttributes, String dossierTemplateId) {
List<com.iqser.red.service.redaction.v1.model.FileAttribute> fileAttributeList = new ArrayList<>();
var fileAttributeConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
for (FileAttributeEntity fileAttribute : fileAttributes) {
var config = fileAttributeConfigs.stream()
.filter(fac -> fac.getId().equals(fileAttribute.getFileAttributeId().getFileAttributeConfigId()))
.findFirst().orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId()));
com.iqser.red.service.redaction.v1.model.FileAttribute attribute = com.iqser.red.service.redaction.v1.model.FileAttribute.builder()
.id(fileAttribute.getFileAttributeConfig().getId())
.label(fileAttribute.getFileAttributeConfig().getLabel())
.placeholder(fileAttribute.getFileAttributeConfig().getPlaceholder())
.id(fileAttribute.getFileAttributeId().getFileAttributeConfigId())
.label(config.getLabel())
.placeholder(config.getPlaceholder())
.value(fileAttribute.getValue())
.build();
fileAttributeList.add(attribute);

9
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java
View File

@ -47,6 +47,7 @@ public class ReanalysisRequiredStatusService {
entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired());
});
return fileModels;
}
@ -87,17 +88,17 @@ public class ReanalysisRequiredStatusService {
Map<String, DossierEntity> dossierMap) {
// enhance with dossierTemplateId
DossierEntity dossier;
try {
DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId()));
fileStatus.setDossierTemplateId(dossier.getDossierTemplateId());
dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId()));
} catch (DossierNotFoundException e) {
log.info("Dossier {} was not found, thus analysis is not required", fileStatus.getDossierId());
return new AnalysisRequiredResult(false, false);
}
// get relevant versions
var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(fileStatus.getDossierTemplateId(),
k -> buildVersionData(fileStatus.getDossierTemplateId()));
var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(),
k -> buildVersionData(dossier.getDossierTemplateId()));
var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(),
k -> getDossierVersionData(fileStatus.getDossierId()));

2
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadProcessorService.java
View File

@ -35,7 +35,7 @@ public class DownloadProcessorService {
DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId());
downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING);
var dossier = downloadStatus.getFiles().iterator().next().getDossier();
var dossier = dossierService.getDossierById(downloadStatus.getFiles().iterator().next().getDossierId());
List<String> filenameSortedFileIds = downloadStatus.getFiles()
.stream()

29
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java
View File

@ -4,10 +4,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService;
import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.amqp.core.AmqpAdmin;
@ -17,6 +14,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@ -30,7 +28,6 @@ public class AutomaticAnalysisScheduler {
@Value("${persistence-service.automaticAnalysis.pageFactor:500}")
private int pageFactor;
private final FileStatusService fileStatusService;
private final ReanalysisRequiredStatusService reanalysisRequiredStatusService;
private final AmqpAdmin amqpAdmin;
@ -53,7 +50,7 @@ public class AutomaticAnalysisScheduler {
var consumerCount = redactionQueueInfo.getConsumerCount();
if (redactionQueueInfo.getMessageCount() <= consumerCount) {
// queue up 5 files
var allStatuses = getAllStatuses();
var allStatuses = getAllRelevantStatuses();
var allStatusesIterator = allStatuses.iterator();
log.info("Files that require reanalysis: {}", allStatuses.size());
@ -80,27 +77,9 @@ public class AutomaticAnalysisScheduler {
}
private List<FileModel> getAllStatuses() {
return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses()
.stream()
.filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED))
.filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED))
.filter(f -> !f.isExcludedFromAutomaticAnalysis())
.filter(f -> !isProcessing(f))
.filter(f -> f.getDossier().getSoftDeletedTime() == null && f.getDossier().getHardDeletedTime() == null && f.getDossier().getArchivedTime() == null)
.collect(Collectors.toList()), FileModel.class, new FileModelMapper()))
.stream()
.filter(FileModel::isAnalysisRequired)
.collect(Collectors.toList());
private List<FileModel> getAllRelevantStatuses() {
return fileStatusService.getAllRelevantStatusesForReanalysisScheduler();
}
private boolean isProcessing(FileEntity file) {
return !file.getProcessingStatus().equals(ProcessingStatus.PROCESSED) && !file.getProcessingStatus()
.equals(ProcessingStatus.UNPROCESSED) && !file.getProcessingStatus()
.equals(ProcessingStatus.DELETED) && !file.getProcessingStatus().equals(ProcessingStatus.ERROR);
}
}

12
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java
View File

@ -40,13 +40,13 @@ public class DeletedFilesCleanupService {
log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId());
}
} else {
var fileEntities = fileStatusService.getDossierStatus(dossierEntity.getId());
for (FileEntity fileEntity : fileEntities) {
if (fileEntity.getHardDeletedTime() == null && fileEntity.getDeleted() != null && fileEntity.getDeleted()
var files = fileStatusService.getDossierStatus(dossierEntity.getId());
for (var file : files) {
if (file.getHardDeletedTime() == null && file.getDeleted() != null && file.getDeleted()
.isBefore(now.minusHours(settings.getSoftDeleteCleanupTime()))) {
fileService.hardDeleteFile(dossierEntity.getId(), fileEntity.getId());
fileStatusService.setFileStatusHardDeleted(fileEntity.getId());
log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), fileEntity.getId());
fileService.hardDeleteFile(dossierEntity.getId(), file.getId());
fileStatusService.setFileStatusHardDeleted(file.getId());
log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), file.getId());
}
}
}

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileModelMapper.java
View File

@ -14,8 +14,5 @@ public class FileModelMapper implements BiConsumer<FileEntity, FileModel> {
fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue())
);
// Without this dossierTemplateId is not set for soft-deleted documents calling get softdeleted with curl.
// however in the unittest it always null without it. Strange....
fileModel.setDossierTemplateId(fileEntity.getDossier().getDossierTemplateId());
}
}

12
persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml
View File

@ -17,11 +17,23 @@ spring:
username: ${PSQL_USERNAME:redaction}
password: ${PSQL_PASSWORD:redaction}
platform: org.hibernate.dialect.PostgreSQL95Dialect
hikari:
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000
prepStmtCacheSqlLimit: 2048
jpa:
database-platform: org.hibernate.dialect.PostgreSQL95Dialect
hibernate:
ddl-auto: none
naming-strategy: org.hibernate.cfg.ImprovedNamingStrategy
properties:
hibernate:
jdbc:
batch_size: 50
order_inserts: true
order_updates: true
profiles:
active: kubernetes

47
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/7-json-column-mapping.changelog.yaml Normal file
View File

@ -0,0 +1,47 @@
databaseChangeLog:
- changeSet:
id: add-file-excluded-pages-column
author: timo
changes:
- addColumn:
columns:
- column:
name: excluded_pages
type: TEXT
tableName: file
- changeSet:
id: add-dossier-download-file-types-members-approvers
author: timo
changes:
- addColumn:
columns:
- column:
name: members
type: TEXT
- column:
name: approvers
type: TEXT
- column:
name: download_file_types
type: TEXT
tableName: dossier
- changeSet:
id: add-dossier-template-download-file-types-members-approvers
author: timo
changes:
- addColumn:
columns:
- column:
name: download_file_types
type: TEXT
tableName: dossier_template
- changeSet:
id: add-download-status-download-file-types-members-approvers
author: timo
changes:
- addColumn:
columns:
- column:
name: download_file_types
type: TEXT
tableName: download_status

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -11,3 +11,7 @@ databaseChangeLog:
file: db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml
- include:
file: db/changelog/6-dossier-status-table.changelog.yaml
- include:
file: db/changelog/7-json-column-mapping.changelog.yaml
- include:
file: db/changelog/sql/7.1-set-json-fields.sql

27
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/7.1-set-json-fields.sql Normal file
View File

@ -0,0 +1,27 @@
update file set excluded_pages =
(select '['||STRING_AGG(excluded_pages || '', ', ')||']' from file_entity_excluded_pages fe where fe.file_entity_id = file.id group by fe.file_entity_id);
update dossier set members =
(select '['||STRING_AGG( '"'||member_ids || '"', ', ')||']'
from dossier_entity_member_ids dem where dem.dossier_entity_id = dossier.id
group by dem.dossier_entity_id);
update dossier set approvers =
(select '['||STRING_AGG( '"'||approver_ids || '"', ', ')||']'
from dossier_entity_approver_ids dem where dem.dossier_entity_id = dossier.id
group by dem.dossier_entity_id);
update dossier set download_file_types =
(select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']'
from dossier_entity_download_file_types dem where dem.dossier_entity_id = dossier.id
group by dem.dossier_entity_id);
update dossier_template set download_file_types =
(select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']'
from dossier_template_entity_download_file_types dem where dem.dossier_template_entity_id = dossier_template.id
group by dem.dossier_template_entity_id);
update download_status set download_file_types =
(select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']'
from download_status_entity_download_file_types dem where dem.download_status_entity_storage_id = download_status.storage_id
group by dem.download_status_entity_storage_id);

2
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java
View File

@ -50,7 +50,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest {
var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1");
var file2 = fileTesterAndProvider.testAndProvideFile(dossier1, "file2");
assertThat(fileClient.getAllStatuses().size()).isEqualTo(2);
assertThat(fileClient.getDossierStatus(dossier1.getId()).size()).isEqualTo(2);
// alter file 1
var loadedFile1 = fileClient.getFileStatus(dossier1.getId(), file1.getId());
assertThat(loadedFile1.isHasRedactions()).isFalse();

12
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -81,7 +81,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier, filename);
assertThat(fileClient.getAllStatuses().size()).isEqualTo(1);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
assertThat(loadedFile.getFilename()).isEqualTo(filename);
@ -118,7 +118,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(changes.iterator().next().isFileChanges()).isTrue();
assertThat(changes.iterator().next().isDossierChanges()).isTrue();
assertThat(fileClient.getAllStatuses().size()).isEqualTo(1);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId());
@ -219,7 +219,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId());
assertThat(softDeletedFiles.size()).isEqualTo(1);
assertThat(softDeletedFiles.get(0).getDossierTemplateId()).isNotNull();
var activeFiles = fileClient.getDossierStatus(dossier.getId());
assertThat(activeFiles.size()).isEqualTo(0);
@ -255,7 +254,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
String typeId = type.getId();
assertThat(fileClient.getAllStatuses().size()).isEqualTo(1);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), "test.pdf", dossier.getId(), "1");
JSONPrimitive<String> uploadResult = uploadClient.upload(upload);
@ -344,7 +343,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
String typeId = type.getId();
assertThat(fileClient.getAllStatuses().size()).isEqualTo(1);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder().addToDictionary(true)
.addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId).user("user").reason("1").value("test").legalBasis("1").build());
@ -380,9 +379,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
String typeId = type.getId();
assertThat(fileClient.getAllStatuses().size()).isEqualTo(1);
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
fileClient.excludePages(dossierId, fileId, Collections.singleton(1));
assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages().size()).isEqualTo(1);

32
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java
View File

@ -4,7 +4,10 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -285,19 +288,24 @@ public class MigrationTest extends AbstractPersistenceServerServiceTest {
}
});
var files = fileClient.getAllStatuses();
files.forEach(file -> {
var manualRedactions = manualRedactionClient.getManualRedactions(file.getDossierId(), file.getId());
if (file.getId().equals("a7985c094cac62118c9b350461902f95")) {
assertThat(manualRedactions.getEntriesToAdd().size()).isEqualTo(1);
assertThat(manualRedactions.getForceRedactions().size()).isEqualTo(1);
assertThat(manualRedactions.getImageRecategorization().size()).isEqualTo(0);
assertThat(manualRedactions.getLegalBasisChanges().size()).isEqualTo(1);
assertThat(manualRedactions.getIdsToRemove().size()).isEqualTo(1);
assertThat(manualRedactions.getComments().size()).isEqualTo(1);
}
List<FileModel> allFiles = new ArrayList<>();
dossiers.forEach(dossier ->{
var files = fileClient.getDossierStatus(dossier.getId());
files.forEach(file -> {
var manualRedactions = manualRedactionClient.getManualRedactions(file.getDossierId(), file.getId());
if (file.getId().equals("a7985c094cac62118c9b350461902f95")) {
assertThat(manualRedactions.getEntriesToAdd().size()).isEqualTo(1);
assertThat(manualRedactions.getForceRedactions().size()).isEqualTo(1);
assertThat(manualRedactions.getImageRecategorization().size()).isEqualTo(0);
assertThat(manualRedactions.getLegalBasisChanges().size()).isEqualTo(1);
assertThat(manualRedactions.getIdsToRemove().size()).isEqualTo(1);
assertThat(manualRedactions.getComments().size()).isEqualTo(1);
}
});
allFiles.addAll(files);
});
assertThat(files.size()).isEqualTo(33);
assertThat(allFiles.size()).isEqualTo(33);
var digitalSignature = digitalSignatureClient.getDigitalSignature();
assertThat(digitalSignature.getCertificateName()).isNotEmpty();

221
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java Normal file
View File

@ -0,0 +1,221 @@
package com.iqser.red.service.peristence.v1.server.integration.tests.performance;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatsClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import lombok.extern.slf4j.Slf4j;
import org.assertj.core.util.Sets;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.time.OffsetDateTime;
import java.util.*;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@Slf4j
public class FilePerformanceTest extends AbstractPersistenceServerServiceTest {
private final static Integer DOSSIER_COUNT = 1;
private final static Integer FILE_PER_DOSSIER_COUNT = 100;
private final static Integer TYPE_COUNT = 1;
private final static Integer TYPE_ENTRY_COUNT = 500;
private final static Integer FILE_ATTRIBUTE_COUNT = 15;
@Autowired
private DossierClient dossierClient;
@Autowired
private FileClient fileClient;
@Autowired
private DossierStatsClient dossierStatsClient;
@Before
public void setupTest() {
long start = System.currentTimeMillis();
DossierTemplateEntity dte = new DossierTemplateEntity();
dte.setId(UUID.randomUUID().toString());
dte.setName("Test");
dte.setDownloadFileTypes(Sets.newTreeSet(DownloadFileType.ANNOTATED, DownloadFileType.REDACTED, DownloadFileType.ORIGINAL, DownloadFileType.PREVIEW, DownloadFileType.FLATTEN));
dte.setCreatedBy("123");
dte.setDescription("Lorem Ipsum");
dte.setModifiedBy("123");
dte = dossierTemplateRepository.save(dte);
List<FileAttributeConfigEntity> fces = new ArrayList<>();
for (int i = 0; i < FILE_ATTRIBUTE_COUNT; i++) {
FileAttributeConfigEntity fce = new FileAttributeConfigEntity();
fce.setId(UUID.randomUUID().toString());
fce.setDossierTemplate(dte);
fce.setLabel("123");
fce.setCsvColumnHeader("123");
fce.setPrimaryAttribute(true);
fce.setFilterable(true);
fce.setPlaceholder("Lorem Ipsum");
fce.setType(FileAttributeType.TEXT);
fce.setDossierTemplate(dte);
fces.add(fce);
}
fces = fileAttributeConfigRepository.saveAll(fces);
log.info("Created File Attributes Config");
for (int i = 0; i < TYPE_COUNT; i++) {
TypeEntity te = new TypeEntity();
te.setId(UUID.randomUUID().toString());
te.setDossierTemplate(dte);
te.setDescription("Lorem Ipsum");
te.setLabel("Lorem ipsum");
te.setType("Type " + i);
te.setAddToDictionaryAction(true);
te.setCaseInsensitive(true);
te.setHexColor("hexColor");
te.setRank(10 + i);
te.setVersion(1231);
te.setRecommendation(true);
te.setHint(false);
te = typeRepository.save(te);
List<DictionaryEntryEntity> des = new ArrayList<>();
for (int j = 0; j < TYPE_ENTRY_COUNT; j++) {
DictionaryEntryEntity de = new DictionaryEntryEntity();
de.setType(te);
de.setDeleted(false);
de.setVersion(1);
de.setValue("Dictionary Entry" + i);
des.add(de);
}
entryRepository.saveAll(des);
log.info("Created Type: {}", te.getType());
}
log.info("Created Dossier Template Data ...");
for (int i = 0; i < DOSSIER_COUNT; i++) {
DossierEntity d = new DossierEntity();
d.setId(UUID.randomUUID().toString());
d.setDossierName("Dossier " + i);
d.setDossierTemplate(dte);
d.setDueDate(OffsetDateTime.now());
d.setStatus(DossierStatus.ACTIVE);
d.setDescription("Lorem Ipsum");
d.setWatermarkEnabled(true);
d.setStartDate(OffsetDateTime.now());
d.setApproverIds(Sets.newTreeSet("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
d.setMemberIds(Sets.newTreeSet("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
d.setDownloadFileTypes(Sets.newTreeSet(DownloadFileType.ANNOTATED, DownloadFileType.REDACTED, DownloadFileType.ORIGINAL, DownloadFileType.PREVIEW, DownloadFileType.FLATTEN));
d = this.dossierRepository.save(d);
List<FileEntity> files = new ArrayList<>();
for (int j = 0; j < FILE_PER_DOSSIER_COUNT; j++) {
FileEntity f = new FileEntity();
f.setId(UUID.randomUUID().toString());
f.setAdded(OffsetDateTime.now());
f.setAssignee("123");
f.setApprovalDate(OffsetDateTime.now());
f.setAnalysisDuration(123);
f.setDictionaryVersion(33);
f.setDossierId(d.getId());
f.setDossierDictionaryVersion(22);
f.setExcluded(false);
f.setExcludedPages(getExcludedPages());
f.setFilename("Lorem Ipsum");
f.setHasAnnotationComments(false);
f.setHasRedactions(true);
f.setHasImages(true);
f.setHasHints(true);
f.setUploader("!2321");
f.setWorkflowStatus(WorkflowStatus.APPROVED);
f.setProcessingStatus(ProcessingStatus.FULLREPROCESS);
f.setLastUpdated(OffsetDateTime.now());
f.setLastUploaded(OffsetDateTime.now());
f.setLastProcessed(OffsetDateTime.now());
f.setLastFileAttributeChange(OffsetDateTime.now());
f.setLastManualRedaction(OffsetDateTime.now());
f.setDossierId(d.getId());
files.add(f);
}
log.info("Created Dossier: {}", d.getDossierName());
files = fileRepository.saveAll(files);
ArrayList<FileAttributeEntity> fileAttributes = new ArrayList<>();
List<FileAttributeConfigEntity> finalFces = fces;
files.forEach(file -> finalFces.forEach(fce -> {
FileAttributeEntity fa = new FileAttributeEntity();
fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId());
fa.getFileAttributeId().setFileId(file.getId());
fa.getFileAttributeId().setFileAttributeConfigId(fce.getId());
fa.setValue("Lorem Ipsum ...");
fileAttributes.add(fa);
}));
fileAttributesRepository.saveAll(fileAttributes);
}
long end = System.currentTimeMillis();
log.info("Created Performance data in: {}ms", end - start);
}
@Test
public void doPerformanceTest() {
long startAllDossiers = System.currentTimeMillis();
var allDossiers = dossierClient.getAllDossiers(false, false);
log.info("Load all dossiers time: {}ms", System.currentTimeMillis() - startAllDossiers);
assertThat(allDossiers.size()).isEqualTo(DOSSIER_COUNT);
long startDossierStats = System.currentTimeMillis();
var dossierStats = dossierStatsClient.getDossierStats(allDossiers.stream().map(Dossier::getId).collect(Collectors.toSet()));
log.info("Load all dossiers stats time: {}ms", System.currentTimeMillis() - startDossierStats);
assertThat(dossierStats.size()).isEqualTo(DOSSIER_COUNT);
for (var dossier : allDossiers) {
long startFilesTime = System.currentTimeMillis();
var files = fileClient.getDossierStatus(dossier.getId());
assertThat(files.size()).isEqualTo(FILE_PER_DOSSIER_COUNT);
log.info("Load files for dossier {} time: {}ms", dossier.getId(), System.currentTimeMillis() - startFilesTime);
}
}
private Set<Integer> getExcludedPages() {
Set<Integer> pages = new HashSet<>();
for (int i = 0; i < 100; i++) {
pages.add(i);
}
return pages;
}
}

58
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -48,7 +48,8 @@ import java.util.TimeZone;
@EnableFeignClients(basePackageClasses = FileClient.class)
@Import(AbstractPersistenceServerServiceTest.TestConfiguration.class)
@ContextConfiguration(initializers = {AbstractPersistenceServerServiceTest.Initializer.class})
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT,
properties = "spring-hibernate-query-utils.n-plus-one-queries-detection.error-level=INFO")
public abstract class AbstractPersistenceServerServiceTest {
@MockBean
@ -62,61 +63,63 @@ public abstract class AbstractPersistenceServerServiceTest {
@MockBean
protected PDFTronRedactionClient pdfTronRedactionClient;
@Autowired
private StorageService storageService;
protected StorageService storageService;
@Autowired
private DossierTemplateRepository dossierTemplateRepository;
protected DossierTemplateRepository dossierTemplateRepository;
@Autowired
private DossierRepository dossierRepository;
protected DossierRepository dossierRepository;
@Autowired
protected FileRepository fileRepository;
@Autowired
private ViewedPagesRepository viewedPagesRepository;
protected ViewedPagesRepository viewedPagesRepository;
@Autowired
private NotificationRepository notificationRepository;
protected NotificationRepository notificationRepository;
@Autowired
private AuditRepository auditRepository;
protected AuditRepository auditRepository;
@Autowired
private TypeRepository typeRepository;
protected TypeRepository typeRepository;
@Autowired
private ManualRedactionRepository manualRedactionRepository;
protected ManualRedactionRepository manualRedactionRepository;
@Autowired
private ForceRedactionRepository forceRedactionRepository;
protected ForceRedactionRepository forceRedactionRepository;
@Autowired
private RemoveRedactionRepository removeRedactionRepository;
protected RemoveRedactionRepository removeRedactionRepository;
@Autowired
private LegalBasisChangeRepository legalBasisChangeRepository;
protected LegalBasisChangeRepository legalBasisChangeRepository;
@Autowired
private ImageRecategorizationRepository imageRecategorizationRepository;
protected ImageRecategorizationRepository imageRecategorizationRepository;
@Autowired
private WatermarkRepository watermarkRepository;
protected WatermarkRepository watermarkRepository;
@Autowired
private SMTPRepository smtpRepository;
protected SMTPRepository smtpRepository;
@Autowired
private RuleSetRepository ruleSetRepository;
protected RuleSetRepository ruleSetRepository;
@Autowired
private LegalBasisMappingRepository legalBasisMappingRepository;
protected LegalBasisMappingRepository legalBasisMappingRepository;
@Autowired
private FileAttributeConfigRepository fileAttributeConfigRepository;
protected FileAttributeConfigRepository fileAttributeConfigRepository;
@Autowired
private FileAttributesGeneralConfigurationRepository fileAttributesGeneralConfigurationRepository;
protected FileAttributesGeneralConfigurationRepository fileAttributesGeneralConfigurationRepository;
@Autowired
private ReportTemplateRepository reportTemplateRepository;
protected ReportTemplateRepository reportTemplateRepository;
@Autowired
private DigitalSignatureRepository digitalSignatureRepository;
protected DigitalSignatureRepository digitalSignatureRepository;
@Autowired
private FileAttributesRepository fileAttributesRepository;
protected FileAttributesRepository fileAttributesRepository;
@Autowired
private DownloadStatusRepository downloadStatusRepository;
protected DownloadStatusRepository downloadStatusRepository;
@Autowired
private DossierAttributeRepository dossierAttributeRepository;
protected DossierAttributeRepository dossierAttributeRepository;
@Autowired
private DossierAttributeConfigRepository dossierAttributeConfigRepository;
protected DossierAttributeConfigRepository dossierAttributeConfigRepository;
@Autowired
private NotificationPreferencesRepository notificationPreferencesRepository;
protected NotificationPreferencesRepository notificationPreferencesRepository;
@Autowired
private DossierStatusRepository dossierStatusRepository;
@Autowired
private MetricsPrinterService metricsPrinterService;
protected MetricsPrinterService metricsPrinterService;
@Autowired
protected EntryRepository entryRepository;
@Before
public void setupOptimize() {
@ -162,6 +165,7 @@ public abstract class AbstractPersistenceServerServiceTest {
@After
public void afterTests() {
entryRepository.deleteAll();
dossierAttributeRepository.deleteAll();
dossierAttributeConfigRepository.deleteAll();
downloadStatusRepository.deleteAll();

7
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java
View File

@ -17,9 +17,16 @@ public class MetricsPrinterService {
@Value("${server.port}")
private int serverPort;
@Value("${management.endpoint.metrics.enabled}")
private boolean metricsEnabled;
@SneakyThrows
public void printMetrics() {
if(!metricsEnabled){
return;
}
RestTemplate rt = new RestTemplate();
var url = "http://127.0.0.1:" + serverPort + "/actuator/metrics";

16
persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml
View File

@ -5,13 +5,23 @@ spring:
datasource:
driverClassName: org.postgresql.Driver
platform: org.hibernate.dialect.PostgreSQL95Dialect
hikari:
data-source-properties:
cachePrepStmts: true
prepStmtCacheSize: 1000
prepStmtCacheSqlLimit: 2048
jpa:
database-platform: org.hibernate.dialect.PostgreSQL95Dialect
hibernate:
ddl-auto: none
naming-strategy: org.hibernate.cfg.ImprovedNamingStrategy
generate-ddl: false
properties:
hibernate:
jdbc:
batch_size: 50
order_inserts: true
order_updates: true
rabbitmq:
host: ${RABBITMQ_HOST:localhost}
@ -59,10 +69,10 @@ persistence-service:
metrics:
persistence:
enabled: true
enabled: false
management:
endpoint:
metrics.enabled: true
metrics.enabled: false
health.enabled: true
endpoints.web.exposure.include: prometheus, health, metrics