diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java index d0ca99c3c..6466a2c74 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/DossierStats.java @@ -2,10 +2,7 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import lombok.*; import java.util.Map; @@ -13,6 +10,7 @@ import java.util.Map; @Builder @NoArgsConstructor @AllArgsConstructor +@EqualsAndHashCode(of ={ "dossierId"}) public class DossierStats { private String dossierId; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java index 33be74688..b70292023 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileModel.java @@ -59,11 +59,13 @@ public class FileModel { private Set excludedPages = new HashSet<>(); private Map fileAttributes = new HashMap<>(); private String dossierId; - private String dossierTemplateId; private OffsetDateTime annotationModificationDate; public boolean isAnalysisRequired(){ return this.fullAnalysisRequired || this.reanalysisRequired; } + public boolean isSoftOrHardDeleted() { + return deleted != null || hardDeletedTime != null || ProcessingStatus.DELETED.equals(processingStatus); + } } diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java index 1efeebfac..d35e56509 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/StatusResource.java @@ -31,11 +31,6 @@ public interface StatusResource { String USER_ID_REQUEST_PARAM = "userId"; - @ResponseBody - @ResponseStatus(value = HttpStatus.OK) - @GetMapping(value = STATUS_PATH, produces = MediaType.APPLICATION_JSON_VALUE) - List getAllStatuses(); - @ResponseBody @ResponseStatus(value = HttpStatus.OK) @PostMapping(value = STATUS_PATH+DOSSIER_ID_PATH_PARAM+CHANGES_PATH, produces = MediaType.APPLICATION_JSON_VALUE) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java index 31de806dc..c81b9893d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualRedactionEntryEntity.java @@ -6,6 +6,9 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.BatchSize; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.time.OffsetDateTime; @@ -51,9 +54,10 @@ public class ManualRedactionEntryEntity implements IBaseAnnotation { private OffsetDateTime softDeletedTime; @ElementCollection + @Fetch(FetchMode.SUBSELECT) private List positions = new ArrayList<>(); - @ManyToOne + @ManyToOne(fetch = FetchType.LAZY) private FileEntity fileStatus; @Column diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java index 6bb03cc35..9faafee68 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/annotations/ManualResizeRedactionEntity.java @@ -6,6 +6,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.time.OffsetDateTime; @@ -38,10 +40,11 @@ public class ManualResizeRedactionEntity implements IBaseAnnotation { @Column(length = 4000) private String value; - @ManyToOne + @ManyToOne(fetch = FetchType.LAZY) private FileEntity fileStatus; @ElementCollection + @Fetch(FetchMode.SUBSELECT) private List positions = new ArrayList<>(); @Column diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java index 8015cd942..fd43a8d55 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/audit/AuditEntity.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.audit; -import com.iqser.red.service.persistence.management.v1.processor.utils.JSONConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -37,7 +38,7 @@ public class AuditEntity { @Basic(fetch = FetchType.EAGER) @Column(columnDefinition = "text") - @Convert(converter = JSONConverter.class) + @Convert(converter = JSONMapConverter.class) private Map details = new HashMap<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java index 145a3556a..8e4941da9 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/LegalBasisMappingEntity.java @@ -4,6 +4,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.util.ArrayList; @@ -23,6 +25,7 @@ public class LegalBasisMappingEntity { private long version; @ElementCollection + @Fetch(FetchMode.SUBSELECT) private List legalBasis = new ArrayList<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java index 137b2edc9..f350f7fb0 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/TypeEntity.java @@ -7,6 +7,9 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.BatchSize; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.util.ArrayList; @@ -45,19 +48,19 @@ public class TypeEntity { @Column private boolean addToDictionaryAction; + @Fetch(FetchMode.SELECT) + @BatchSize(size=500) @OneToMany(cascade = ALL, mappedBy = "type", orphanRemoval = true, fetch = FetchType.LAZY) private List entries = new ArrayList<>(); - @ManyToOne - @JsonIgnore + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "dossier_template_id") private DossierTemplateEntity dossierTemplate; @Column(updatable = false, insertable = false, name = "dossier_template_id") private String dossierTemplateId; - @ManyToOne - @JsonIgnore + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "dossier_id") private DossierEntity dossier; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java index 6e5b0f037..255eceb2d 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/WatermarkEntity.java @@ -7,6 +7,9 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.BatchSize; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; @@ -31,11 +34,11 @@ public class WatermarkEntity { private int fontSize; @Column private String fontType; + @Column @Enumerated(EnumType.STRING) private WatermarkOrientation orientation; - @JsonIgnore @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "dossier_template_id") private DossierTemplateEntity dossierTemplate; diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java index 1abf09634..9b881db93 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierAttributeEntity.java @@ -25,16 +25,6 @@ public class DossierAttributeEntity { @Column(columnDefinition = "TEXT") private String value; - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - @MapsId("dossierId") - private DossierEntity dossier; - - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - @MapsId("dossierAttributeConfigId") - private DossierAttributeConfigEntity dossierAttributeConfig; - @Data @NoArgsConstructor @AllArgsConstructor diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java index b13273cab..1d79904ff 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierEntity.java @@ -2,12 +2,13 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import lombok.*; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.time.OffsetDateTime; @@ -22,6 +23,7 @@ import java.util.Set; @NoArgsConstructor @AllArgsConstructor @Table(name = "dossier") +@EqualsAndHashCode(of = {"id"}) public class DossierEntity { @Id @@ -46,16 +48,18 @@ public class DossierEntity { private String ownerId; @Builder.Default - @ElementCollection + @Column(columnDefinition = "text", name = "members") + @Convert(converter = JSONStringSetConverter.class) private Set memberIds = new HashSet<>(); @Builder.Default - @ElementCollection + @Column(columnDefinition = "text", name = "approvers") + @Convert(converter = JSONStringSetConverter.class) private Set approverIds = new HashSet<>(); @Builder.Default - @ElementCollection - @Enumerated(EnumType.STRING) + @Column(columnDefinition = "text", name = "download_file_types") + @Convert(converter = JSONDownloadFileTypeConverter.class) private Set downloadFileTypes = new HashSet<>(); @Column @@ -87,15 +91,19 @@ public class DossierEntity { private String dossierTemplateId; @Builder.Default + @Fetch(FetchMode.SUBSELECT) @ManyToMany(fetch = FetchType.LAZY, mappedBy = "dossiers") private List reportTemplates = new ArrayList<>(); @Builder.Default - @OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier") + @Fetch(FetchMode.SUBSELECT) + @OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier", fetch = FetchType.LAZY) private List dossierTypes = new ArrayList<>(); @Builder.Default - @OneToMany(cascade = CascadeType.ALL, mappedBy = "dossier") + @Fetch(FetchMode.SUBSELECT) + @JoinColumn(name = "dossier_id") + @OneToMany(cascade = CascadeType.ALL, fetch = FetchType.LAZY) private List dossierAttributes = new ArrayList<>(); @ManyToOne(fetch = FetchType.LAZY) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java index c411c287a..0e8d77cc7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierStatusEntity.java @@ -4,6 +4,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.hibernate.annotations.Formula; import javax.persistence.*; @@ -37,7 +39,8 @@ public class DossierStatusEntity { @Column(updatable = false, insertable = false, name = "dossier_template_id") private String dossierTemplateId; - @OneToMany(mappedBy = "dossierStatus") + @OneToMany(mappedBy = "dossierStatus", fetch = FetchType.LAZY) + @Fetch(FetchMode.SUBSELECT) private List dossiers = new ArrayList<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java index c9398f5e6..a453fc3f1 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/DossierTemplateEntity.java @@ -6,18 +6,11 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Id; -import javax.persistence.OneToMany; -import javax.persistence.Table; -import javax.persistence.Transient; +import javax.persistence.*; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; @@ -25,6 +18,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; @Data @Entity @@ -64,31 +59,37 @@ public class DossierTemplateEntity { @Column(name = "soft_delete_time") private OffsetDateTime softDeleteTime; - @ElementCollection - @Enumerated(EnumType.STRING) + @Builder.Default + @Column(columnDefinition = "text", name = "download_file_types") + @Convert(converter = JSONDownloadFileTypeConverter.class) private Set downloadFileTypes = new HashSet<>(); - @JsonIgnore + @Builder.Default @OneToMany(mappedBy = "dossierTemplate") + @Fetch(FetchMode.SUBSELECT) private List reportTemplates = new ArrayList<>(); - @JsonIgnore + @Builder.Default @OneToMany(mappedBy = "dossierTemplate") + @Fetch(FetchMode.SUBSELECT) private List dossierTypes = new ArrayList<>(); - @JsonIgnore + @Builder.Default @OneToMany(mappedBy = "dossierTemplate") + @Fetch(FetchMode.SUBSELECT) private List dossiers = new ArrayList<>(); - @JsonIgnore + @Builder.Default + @Fetch(FetchMode.SUBSELECT) @OneToMany(mappedBy = "dossierTemplate") private List fileAttributeConfigs = new ArrayList<>(); + @Builder.Default + @Fetch(FetchMode.SUBSELECT) + @OneToMany(mappedBy = "dossierTemplate") + private List dossierStatusList = new ArrayList<>(); + @Transient private DossierTemplateStatus dossierTemplateStatus; - @JsonIgnore - @OneToMany(mappedBy = "dossierTemplate") - private List dossierStatusList = new ArrayList<>(); - } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java index 51053ad64..a9e9743e6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileAttributeEntity.java @@ -1,6 +1,5 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import javax.persistence.*; @@ -17,17 +16,6 @@ public class FileAttributeEntity { @Column(length = 4000) private String value; - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - @MapsId("fileId") - private FileEntity file; - - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - @MapsId("fileAttributeConfigId") - private FileAttributeConfigEntity fileAttributeConfig; - - @Data @Embeddable public static class FileAttributeEntityId implements Serializable { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java index 5cd4bcabf..ece5e7f87 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/dossier/FileEntity.java @@ -1,12 +1,14 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.dossier; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONIntegerSetConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.time.OffsetDateTime; @@ -133,17 +135,16 @@ public class FileEntity { @Column private OffsetDateTime annotationModificationDate; - @ElementCollection(fetch = FetchType.EAGER) + @Column(columnDefinition = "text", name = "excluded_pages") + @Convert(converter = JSONIntegerSetConverter.class) private Set excludedPages = new HashSet<>(); - @OneToMany(mappedBy = "file", fetch = FetchType.EAGER, cascade = CascadeType.ALL) + @JoinColumn(name = "file_id") + @Fetch(FetchMode.SUBSELECT) + @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL) private List fileAttributes = new ArrayList<>(); - @JsonIgnore - @ManyToOne - private DossierEntity dossier; - - @Column(updatable = false, insertable = false, name = "dossier_id") + @Column(name = "dossier_id") private String dossierId; public boolean isSoftDeleted() { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java index f94e97d19..460568f75 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/download/DownloadStatusEntity.java @@ -2,17 +2,23 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.downloa import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONDownloadFileTypeConverter; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.download.DownloadStatusValue; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.BatchSize; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; @Data @Entity @@ -40,15 +46,17 @@ public class DownloadStatusEntity { @Column private long fileSize; - @ManyToOne + @ManyToOne(fetch = FetchType.LAZY) private DossierEntity dossier; @ManyToMany + @Fetch(FetchMode.SUBSELECT) private List files = new ArrayList<>(); - @ElementCollection - @Enumerated(EnumType.STRING) - private List downloadFileTypes = new ArrayList<>(); + @Builder.Default + @Column(columnDefinition = "text", name = "download_file_types") + @Convert(converter = JSONDownloadFileTypeConverter.class) + private Set downloadFileTypes = new HashSet<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java index ae25a69aa..23ced3227 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationEntity.java @@ -1,6 +1,7 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.notification; -import com.iqser.red.service.persistence.management.v1.processor.utils.JSONConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONMapConverter; +import com.iqser.red.service.persistence.management.v1.processor.utils.JSONStringSetConverter; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -49,6 +50,6 @@ public class NotificationEntity { @Basic(fetch = FetchType.EAGER) @Column(columnDefinition = "text") - @Convert(converter = JSONConverter.class) + @Convert(converter = JSONMapConverter.class) private Map target = new HashMap<>(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java index b0f48b6cd..d2c8ec47a 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/notification/NotificationPreferencesEntity.java @@ -5,6 +5,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import javax.persistence.*; import java.util.ArrayList; @@ -31,8 +33,10 @@ public class NotificationPreferencesEntity { private EmailNotificationType emailNotificationType; @ElementCollection + @Fetch(FetchMode.SUBSELECT) private List emailNotifications = new ArrayList<>(); @ElementCollection + @Fetch(FetchMode.SUBSELECT) private List inAppNotifications = new ArrayList<>(); -} \ No newline at end of file +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java index bce14765b..58f8bb88f 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DossierAttributePersistenceService.java @@ -24,11 +24,9 @@ public class DossierAttributePersistenceService { public void insertDossierAttribute(String dossierId, String dossierAttributeId, String dossierAttributeValue) { DossierAttributeEntity dossierAttribute = new DossierAttributeEntity(); - dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(null, null)); + dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(dossierId, dossierAttributeId)); dossierAttribute.setValue(dossierAttributeValue); - dossierAttribute.setDossier(dossierRepository.getOne(dossierId)); - dossierAttribute.setDossierAttributeConfig(dossierAttributeConfigRepository.getOne(dossierAttributeId)); - dossierAttributeRepository.save(dossierAttribute); + dossierAttributeRepository.saveAndFlush(dossierAttribute); } @Transactional diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java index 43aa5b59f..f20e066d6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DownloadStatusPersistenceService.java @@ -14,6 +14,7 @@ import javax.transaction.Transactional; import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -36,7 +37,7 @@ public class DownloadStatusPersistenceService { downloadStatus.setDossier(dossier); downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); downloadStatus.setFiles(fileRepository.findAllById(fileIds)); - downloadStatus.setDownloadFileTypes(new ArrayList<>(downloadFileTypes)); + downloadStatus.setDownloadFileTypes(new HashSet<>(downloadFileTypes)); downloadStatusRepository.save(downloadStatus); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java index 421548a22..9b343f271 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/FileStatusPersistenceService.java @@ -37,7 +37,7 @@ public class FileStatusPersistenceService { FileEntity file = new FileEntity(); file.setId(fileId); - file.setDossier(dossierRepository.getOne(dossierId)); + file.setDossierId(dossierId); file.setFilename(filename); file.setProcessingStatus(ProcessingStatus.UNPROCESSED); file.setWorkflowStatus(WorkflowStatus.NEW); @@ -58,8 +58,9 @@ public class FileStatusPersistenceService { if (isFileDeleted(fileId)) { return; } - fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now() - .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber); + fileRepository.updateProcessingStatus(fileId, numberOfPages, ProcessingStatus.PROCESSED, dictionaryVersion, rulesVersion, + legalBasisVersion, duration, dossierDictionaryVersion, analysisVersion, OffsetDateTime.now() + .truncatedTo(ChronoUnit.MILLIS), OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS), analysisNumber); } @@ -167,9 +168,9 @@ public class FileStatusPersistenceService { } var fileAttributeEntities = convertFileAttributes(dossierId, file, fileAttributes); - file.setLastFileAttributeChange(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - file.setLastUpdated(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); - file.setFileAttributes(fileAttributeEntities); + fileAttributesRepository.saveAllAndFlush(fileAttributeEntities); + fileRepository.updateLastAttributeChangeDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); + }, () -> { throw new NotFoundException("Unknown file=" + fileId); }); @@ -178,8 +179,7 @@ public class FileStatusPersistenceService { @Transactional - public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, - boolean hasSuggestions) { + public void setUpdateLastManualRedactionAndHasSuggestions(String fileId, OffsetDateTime date, boolean hasSuggestions) { if (isFileDeleted(fileId)) { return; @@ -220,12 +220,6 @@ public class FileStatusPersistenceService { } - public List getAllStatuses() { - - return fileRepository.findAll(); - } - - public List getActiveFiles(String dossierId) { return fileRepository.findByDossierId(dossierId) @@ -339,12 +333,13 @@ public class FileStatusPersistenceService { return fileAttributesMap.entrySet().stream().map(entry -> { var fa = new FileAttributeEntity(); - fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId()); - fa.setFile(file); - fa.setFileAttributeConfig(configuration.stream() + var id = new FileAttributeEntity.FileAttributeEntityId(); + id.setFileId(file.getId()); + id.setFileAttributeConfigId(configuration.stream() .filter(c -> c.getId().equals(entry.getKey())) - .findAny() + .findAny().map(FileAttributeConfigEntity::getId) .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id"))); + fa.setFileAttributeId(id); fa.setValue(entry.getValue()); return fa; }).collect(Collectors.toList()); @@ -369,4 +364,7 @@ public class FileStatusPersistenceService { return fileRepository.countSoftDeletedFiles(dossierId); } + public List getAllRelevantStatusesForReanalysisScheduler() { + return fileRepository.getAllRelevantStatusesForReanalysisScheduler(); + } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java index ceb381600..fc268e023 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/DossierAttributeRepository.java @@ -11,12 +11,11 @@ public interface DossierAttributeRepository extends JpaRepository findByIdDossierId(String dossierId); + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("DELETE FROM DossierAttributeEntity e WHERE e.id.dossierId = :id") void deleteByDossierId(String dossierId); - @Query("SELECT a FROM DossierAttributeEntity a WHERE a.dossierAttributeConfig.dossierTemplate.id = :dossierTemplateId") - List findByDossierTemplateId(String dossierTemplateId); - - @Modifying + @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("DELETE FROM DossierAttributeEntity e WHERE e.id.dossierAttributeConfigId = :id") void deleteByDossierAttributeConfigId(String id); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java index abba4ef1c..9059b4248 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileAttributesRepository.java @@ -5,13 +5,18 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import java.util.List; + public interface FileAttributesRepository extends JpaRepository { - @Modifying + @Modifying(flushAutomatically = true, clearAutomatically = true) @Query("DELETE FROM FileAttributeEntity f where f.fileAttributeId.fileAttributeConfigId = :fileAttributeConfigId") void deleteByFileAttributeConfigId(String fileAttributeConfigId); - @Modifying + @Modifying(flushAutomatically = true, clearAutomatically = true) @Query("DELETE FROM FileAttributeEntity f where f.fileAttributeId.fileId = :fileId") void deleteByFileId(String fileId); + + @Query("SELECT f FROM FileAttributeEntity f where f.fileAttributeId.fileId = :fileId") + List findByFileId(String fileId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java index accab3459..9c35379db 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FileRepository.java @@ -139,6 +139,15 @@ public interface FileRepository extends JpaRepository { @Modifying(clearAutomatically = true) @Query("update FileEntity f set f.annotationModificationDate = :annotationModificationDate where f.id = :fileId") void setLastAnnotationModificationDateForFile(String fileId, OffsetDateTime annotationModificationDate); + + @Query("select f from FileEntity f join DossierEntity d on d.id = f.dossierId where f.workflowStatus <> 'APPROVED' and f.excludedFromAutomaticAnalysis = false " + + "and ( f.processingStatus = 'PROCESSED' or f.processingStatus = 'UNPROCESSED' or f.processingStatus = 'DELETED' or f.processingStatus = 'ERROR' )" + + "and d.softDeletedTime is null and d.hardDeletedTime is null and d.status = 'ACTIVE' " ) + List getAllRelevantStatusesForReanalysisScheduler(); + + @Modifying(clearAutomatically = true) + @Query("update FileEntity f set f.lastFileAttributeChange = :date, f.lastUpdated = :date") + void updateLastAttributeChangeDate(OffsetDateTime date); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONDownloadFileTypeConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONDownloadFileTypeConverter.java new file mode 100644 index 000000000..ca40c9e92 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONDownloadFileTypeConverter.java @@ -0,0 +1,33 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils; + +import com.amazonaws.services.s3.transfer.Download; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import lombok.SneakyThrows; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import java.util.HashSet; +import java.util.Set; + +@Converter +public class JSONDownloadFileTypeConverter implements AttributeConverter, String> { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @SneakyThrows + @Override + public String convertToDatabaseColumn(Set dataSet) { + return objectMapper.writeValueAsString(dataSet); + } + + @SneakyThrows + @Override + public Set convertToEntityAttribute(String data) { + TypeReference> typeRef = new TypeReference<>() { + }; + return objectMapper.readValue(data, typeRef); + + } +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONIntegerSetConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONIntegerSetConverter.java new file mode 100644 index 000000000..4bb15530e --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONIntegerSetConverter.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import java.util.HashSet; +import java.util.Set; + +@Converter +public class JSONIntegerSetConverter implements AttributeConverter, String> { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @SneakyThrows + @Override + public String convertToDatabaseColumn(Set dataSet) { + return objectMapper.writeValueAsString(dataSet); + } + + @SneakyThrows + @Override + public Set convertToEntityAttribute(String data) { + TypeReference> typeRef = new TypeReference<>() { + }; + return objectMapper.readValue(data, typeRef); + + } +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONMapConverter.java similarity index 90% rename from persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONConverter.java rename to persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONMapConverter.java index 5e20fb791..b0de9f442 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONConverter.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONMapConverter.java @@ -10,11 +10,10 @@ import java.util.HashMap; import java.util.Map; @Converter -public class JSONConverter implements AttributeConverter, String> { +public class JSONMapConverter implements AttributeConverter, String> { private final ObjectMapper objectMapper = new ObjectMapper(); - @SneakyThrows @Override public String convertToDatabaseColumn(Map map) { diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStringSetConverter.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStringSetConverter.java new file mode 100644 index 000000000..aecf91b77 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/JSONStringSetConverter.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; +import java.util.HashSet; +import java.util.Set; + +@Converter +public class JSONStringSetConverter implements AttributeConverter, String> { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @SneakyThrows + @Override + public String convertToDatabaseColumn(Set dataSet) { + return objectMapper.writeValueAsString(dataSet); + } + + @SneakyThrows + @Override + public Set convertToEntityAttribute(String data) { + TypeReference> typeRef = new TypeReference<>() { + }; + return objectMapper.readValue(data, typeRef); + + } +} diff --git a/persistence-service-v1/persistence-service-server-v1/pom.xml b/persistence-service-v1/persistence-service-server-v1/pom.xml index bb40a34bd..e466f986d 100644 --- a/persistence-service-v1/persistence-service-server-v1/pom.xml +++ b/persistence-service-v1/persistence-service-server-v1/pom.xml @@ -18,6 +18,7 @@ + com.iqser.red.service search-service-api-v1 @@ -153,6 +154,12 @@ liquibase-core 4.3.1 + + com.yannbriancon + spring-hibernate-query-utils + 2.0.0 + test + diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java index 9767cbb9d..25eeaa222 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DossierController.java @@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.*; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.resources.DossierResource; import com.iqser.red.service.search.v1.model.IndexMessageType; import feign.Param; @@ -74,7 +75,7 @@ public class DossierController implements DossierResource { throw new DossierNotFoundException(String.format(DOSSIER_NOT_FOUND_MESSAGE, dossierId)); } - List fileStatuses = fileStatusService.getDossierStatus(dossierId); + List fileStatuses = fileStatusService.getDossierStatus(dossierId); fileStatuses.stream().filter(fileStatus -> fileStatus.getDeleted() == null).forEach(fileStatus -> { fileService.softDeleteFile(dossierId, fileStatus.getId(), now); fileStatusService.setFileStatusDeleted(fileStatus.getId(), now); @@ -171,7 +172,7 @@ public class DossierController implements DossierResource { for (String dossierId : dossierIds) { DossierEntity dossier = dossierService.getDossierById(dossierId); dossierService.hardDeleteDossier(dossier.getId()); - List fileStatuses = fileStatusService.getDossierStatus(dossierId); + List fileStatuses = fileStatusService.getDossierStatus(dossierId); fileStatuses.forEach(fileStatus -> { fileService.hardDeleteFile(dossierId, fileStatus.getId()); fileStatusService.setFileStatusHardDeleted(fileStatus.getId()); @@ -186,7 +187,7 @@ public class DossierController implements DossierResource { for (String dossierId : dossierIds) { var dossier = dossierService.getDossierById(dossierId); - List fileStatuses = fileStatusService.getDossierStatus(dossierId); + List fileStatuses = fileStatusService.getDossierStatus(dossierId); fileStatuses.forEach(fileStatus -> { if (fileStatus.getDeleted() != null && (fileStatus.getDeleted().equals(dossier.getSoftDeletedTime()) || fileStatus.getDeleted() .isAfter(dossier.getSoftDeletedTime()))) { @@ -208,7 +209,7 @@ public class DossierController implements DossierResource { DossierEntity dossier = dossierService.getDossierById(dossierId); if (dossier.getHardDeletedTime() == null) { dossierService.archiveDossier(dossier.getId()); - List fileStatuses = fileStatusService.getDossierStatus(dossier.getId()); + List fileStatuses = fileStatusService.getDossierStatus(dossier.getId()); addToIndexingQueue(dossierId, fileStatuses); } else { log.error("Could not archive Dossier {}, because it is already deleted.", dossierId); @@ -225,7 +226,7 @@ public class DossierController implements DossierResource { DossierEntity dossier = dossierService.getDossierById(dossierId); if (dossier.getHardDeletedTime() == null) { dossierService.unarchiveDossier(dossier.getId()); - List fileStatuses = fileStatusService.getDossierStatus(dossier.getId()); + List fileStatuses = fileStatusService.getDossierStatus(dossier.getId()); addToIndexingQueue(dossierId, fileStatuses); } else { log.error("Could not unarchive Dossier {}, because it is deleted.", dossierId); @@ -235,7 +236,7 @@ public class DossierController implements DossierResource { } - private void addToIndexingQueue(String dossierId, List fileStatuses) { + private void addToIndexingQueue(String dossierId, List fileStatuses) { fileStatuses.forEach(f -> indexingService.addToIndexingQueue(IndexMessageType.UPDATE, null, dossierId, f.getId(), 2)); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java index 06e46a870..433d5aea6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileAttributesController.java @@ -14,6 +14,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ImportCsvResponse; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.resources.FileAttributesResource; import com.iqser.red.service.persistence.service.v1.api.utils.SuppressFBWarnings; @@ -60,10 +61,10 @@ public class FileAttributesController implements FileAttributesResource { FileAttributesGeneralConfigurationEntity generalConfiguration = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossier.getDossierTemplateId()); List configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossier.getDossierTemplateId()); - Map fileStatusByFilename = fileStatusService.getDossierStatus(dossierId) + Map fileStatusByFilename = fileStatusService.getDossierStatus(dossierId) .stream() .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) - .collect(Collectors.toMap(FileEntity::getFilename, Function.identity())); + .collect(Collectors.toMap(FileModel::getFilename, Function.identity())); List> rows = getCsvRecords(importCsvRequest.getCsvFile(), generalConfiguration.getDelimiter()); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java index 56584703c..c07076d56 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/FileStatusController.java @@ -12,6 +12,7 @@ import com.iqser.red.service.persistence.service.v1.api.resources.StatusResource import com.iqser.red.service.search.v1.model.IndexMessageType; import lombok.RequiredArgsConstructor; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; @@ -31,18 +32,8 @@ public class FileStatusController implements StatusResource { private final FileStatusService fileStatusService; private final ExcludeFromAnalysisService excludeFromAnalysis; private final AnalysisFlagsCalculationService analysisFlagsCalculationService; - private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final IndexingService indexingService; - @Override - public List getAllStatuses() { - - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses() - .stream() - .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) - .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); - } - @Override public JSONPrimitive hasChangesSince(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @@ -55,31 +46,23 @@ public class FileStatusController implements StatusResource { @Override public List getDossierStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId) { - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId) - .stream() + return fileStatusService.getDossierStatus(dossierId).stream() .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) - .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); + .collect(Collectors.toList()); } @Override public List getSoftDeletedDossierStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId) { - - var softDeletedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getDossierStatus(dossierId) - .stream() - .filter(f -> f.getProcessingStatus().equals(ProcessingStatus.DELETED) && f.getHardDeletedTime() == null) - .collect(Collectors.toList()), FileModel.class, new FileModelMapper())); - softDeletedFiles.sort((f1, f2) -> f2.getDeleted().compareTo(f1.getDeleted())); - - return softDeletedFiles; + return fileStatusService.getDossierStatus(dossierId).stream() + .filter(f3 -> f3.getProcessingStatus().equals(ProcessingStatus.DELETED) && f3.getHardDeletedTime() == null) + .sorted((f11, f21) -> f21.getDeleted().compareTo(f11.getDeleted())).collect(Collectors.toList()); } @Override public FileModel getFileStatus(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements( - convert(fileStatusService.getStatus(fileId), FileModel.class, new FileModelMapper())); + return fileStatusService.getStatus(fileId); } @@ -97,7 +80,7 @@ public class FileStatusController implements StatusResource { @RequestParam(value = USER_ID_REQUEST_PARAM, required = false) String userId) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + FileModel fileStatus = fileStatusService.getStatus(fileId); String assignee = fileStatus.getAssignee(); if (userId != null) { assignee = userId; @@ -112,7 +95,7 @@ public class FileStatusController implements StatusResource { @PathVariable(FILE_ID) String fileId, @RequestParam(value = APPROVER_ID_REQUEST_PARAM, required = false) String approverId) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + FileModel fileStatus = fileStatusService.getStatus(fileId); String assignee = fileStatus.getAssignee(); if (approverId != null) { assignee = approverId; @@ -128,7 +111,7 @@ public class FileStatusController implements StatusResource { @PathVariable(FILE_ID) String fileId, @RequestParam(value = APPROVER_ID_REQUEST_PARAM, required = false) String approverId) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + FileModel fileStatus = fileStatusService.getStatus(fileId); String assignee = fileStatus.getAssignee(); if (assignee == null && approverId == null) { throw new BadRequestException("No user assigned for the approval"); @@ -158,7 +141,8 @@ public class FileStatusController implements StatusResource { public void excludePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set pages) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + + FileModel fileStatus = fileStatusService.getStatus(fileId); Set excludedPages = fileStatus.getExcludedPages(); excludedPages.addAll(pages); @@ -169,7 +153,7 @@ public class FileStatusController implements StatusResource { public void includePages(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId, @RequestBody Set pages) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + FileModel fileStatus = fileStatusService.getStatus(fileId); Set excludedPages = fileStatus.getExcludedPages(); excludedPages.removeAll(pages); fileStatusService.setExcludedPages(fileId, excludedPages); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index f509607d1..a2f73ba3b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -95,7 +95,7 @@ public class ReanalysisController implements ReanalysisResource { if (force) { fileStatusService.setStatusOcrProcessing(dossierId, fileId); } else { - FileEntity dossierFile = fileStatusService.getStatus(fileId); + FileModel dossierFile = fileStatusService.getStatus(fileId); if (dossierFile.getProcessingStatus().equals(ProcessingStatus.DELETED) || dossierFile.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { throw new ConflictException("Cannot analyse a deleted/approved file"); } @@ -107,7 +107,7 @@ public class ReanalysisController implements ReanalysisResource { } - private List getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) { + private List getAllFilesForDossier(String dossierId, boolean filterOnlyValidFiles) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); @@ -124,11 +124,11 @@ public class ReanalysisController implements ReanalysisResource { } - private List getRelevantFiles(String dossierId, Collection fileIds) { + private List getRelevantFiles(String dossierId, Collection fileIds) { var dossierFiles = getAllFilesForDossier(dossierId, false); - var relevantFiles = new ArrayList(); + var relevantFiles = new ArrayList(); for (var fileId : fileIds) { var dossierFileOptional = dossierFiles.stream().filter(f -> f.getId().equals(fileId)).findAny(); if (dossierFileOptional.isEmpty()) { @@ -150,17 +150,14 @@ public class ReanalysisController implements ReanalysisResource { } - private void reanalyseFiles(String dossierId, boolean force, List filesToReanalyse) { + private void reanalyseFiles(String dossierId, boolean force, List filesToReanalyse) { if (force) { filesToReanalyse.forEach(file -> { fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true); }); } else { - var enhancedAndConvertedFiles = reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert( - fileStatusService.getAllStatuses(), FileModel.class, new FileModelMapper())); - - enhancedAndConvertedFiles.forEach(file -> { + filesToReanalyse.stream().filter(FileModel::isReanalysisRequired).forEach(file -> { fileStatusService.setStatusReprocess(dossierId, file.getId(), 2, true); }); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java index d69a9c589..93dd6358e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java @@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimiti import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource; import com.iqser.red.service.redaction.v1.model.AnnotateRequest; @@ -91,7 +92,7 @@ public class UploadController implements UploadResource { for (String fileId : fileIds) { - FileEntity fileStatus = fileStatusService.getStatus(fileId); + FileModel fileStatus = fileStatusService.getStatus(fileId); OffsetDateTime softDeletedTime = fileStatus.getDeleted(); fileService.undeleteFile(dossier.getDossierTemplateId(), dossierId, fileId, softDeletedTime); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java index 352a6ce0b..87761c786 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/MigrationService.java @@ -178,6 +178,7 @@ public class MigrationService { private void migrateRedactionLogs() { List files = fileRepository.findAll(); + List dossiers = dossierRepository.findAll(); files.stream().filter(file -> file.getHardDeletedTime() == null).forEach(file -> { migrateRedactionLog(file); }); @@ -341,7 +342,7 @@ public class MigrationService { downloadStatus.setDossier(dossierRepository.getOne(download.getDossierId())); downloadStatus.setCreationDate(OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)); downloadStatus.setFiles(fileRepository.findAllById(download.getDownloadDetails().getFileIds())); - downloadStatus.setDownloadFileTypes(new ArrayList<>(download.getDownloadDetails().getDownloadFileTypes())); + downloadStatus.setDownloadFileTypes(new HashSet<>(download.getDownloadDetails().getDownloadFileTypes())); downloadStatus.setStatus(download.getStatus()); downloadStatus.setFileSize(download.getFileSize()); downloadStatus.setLastDownload(download.getLastDownload()); @@ -458,12 +459,13 @@ public class MigrationService { imageRedactions.forEach(redaction -> { var file = fileRepository.getById(redaction.getFileId()); + var dossier = dossierRepository.getById(file.getDossierId()); var manualImageRecategorization = ManualImageRecategorizationEntity.builder() .id(new AnnotationEntityId(redaction.getId(), redaction.getFileId())) .user(redaction.getUser()) .status(redaction.getStatus()) - .typeId(toTypeId(redaction.getType(), file.getDossier().getDossierTemplateId(), null)) + .typeId(toTypeId(redaction.getType(), dossier.getDossierTemplateId(), null)) .requestDate(redaction.getRequestDate()) .processedDate(redaction.getProcessedDate()) .softDeletedTime(redaction.getSoftDeletedTime()) @@ -569,14 +571,13 @@ public class MigrationService { addRedactions.forEach(addRedaction -> { var file = fileRepository.getById(addRedaction.getFileId()); - +var dossier = dossierRepository.findById(file.getDossierId()); ManualRedactionEntryEntity manualRedactionEntry = ManualRedactionEntryEntity.builder() .id(new AnnotationEntityId(addRedaction.getId(), addRedaction.getFileId())) .user(addRedaction.getUser()) - .typeId(toTypeId(addRedaction.getType(), file.getDossier() - .getDossierTemplateId(), addRedaction.getType() - .equals("dossier_redaction") ? file.getDossier().getId() : null)) + .typeId(toTypeId(addRedaction.getType(), dossier.get().getDossierTemplateId(), addRedaction.getType() + .equals("dossier_redaction") ? dossier.get().getDossierTemplateId() : null)) .value(addRedaction.getValue()) .reason(addRedaction.getReason()) .legalBasis(addRedaction.getLegalBasis()) @@ -800,14 +801,12 @@ public class MigrationService { .lastFileAttributeChange(oldFile.getLastFileAttributeChange()) .analysisVersion(oldFile.getAnalysisVersion()) .excludedPages(oldFile.getExcludedPages()) - .dossier(dossierEntity) + .dossierId(dossierEntity.getId()) .dossierId(oldFile.getDossierId()) .build(); List convertedFileAttributes = new ArrayList<>(); - List configuration = fileAttributeConfigPersistenceService.getFileAttributes(file - .getDossier() - .getDossierTemplateId()); + List configuration = fileAttributeConfigPersistenceService.getFileAttributes(dossierEntity.getDossierTemplateId()); if (oldFile.getFileAttributes() != null) { oldFile.getFileAttributes().getAttributeIdToValue().entrySet().forEach(entry -> { @@ -816,16 +815,16 @@ public class MigrationService { .stream() .filter(nId -> nId.getDossierTemplateId().equals(dossierEntity.getDossierTemplateId())) .findFirst(); - if (!newId.isPresent()) { + if (newId.isEmpty()) { return; } var fa = new FileAttributeEntity(); fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId()); - fa.setFile(file); - fa.setFileAttributeConfig(configuration.stream() + fa.getFileAttributeId().setFileId(file.getId()); + fa.getFileAttributeId().setFileAttributeConfigId(configuration.stream() .filter(c -> c.getId().equals(newId.get().getNewId())) - .findAny() + .findAny().map(FileAttributeConfigEntity::getId) .orElseThrow(() -> new BadRequestException("Invalid File Attribute Id"))); fa.setValue(entry.getValue()); convertedFileAttributes.add(fa); @@ -860,9 +859,6 @@ public class MigrationService { dossierAttribute.setId(new DossierAttributeEntity.DossierAttributeEntityId(dossier.getId(), newConfigId.get() .getNewId())); dossierAttribute.setValue(oldDossierAttribute.getValue()); - dossierAttribute.setDossier(dossier); - dossierAttribute.setDossierAttributeConfig(dossierAttributeConfigRepository.getOne(newConfigId.get() - .getNewId())); dossierAttributeRepository.save(dossierAttribute); } }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java index 9b4ca672a..44a77c6e0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DossierStatsService.java @@ -5,6 +5,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStats; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; @@ -42,7 +43,12 @@ public class DossierStatsService { } public DossierStats getDossierStats(String dossierId) { - return dossierStatsMap.getOrDefault(dossierId, computeDossierStats(dossierService.getDossierById(dossierId))); + var stats = dossierStatsMap.get(dossierId); + if (stats == null) { + stats = computeDossierStats(dossierService.getDossierById(dossierId)); + dossierStatsMap.put(dossierId, stats); + } + return stats; } private DossierStats computeDossierStats(DossierEntity dossierEntity) { @@ -56,21 +62,21 @@ public class DossierStatsService { dossierStats.setDossierId(dossierId); // get the associated files - List files = fileStatusService.getActiveFiles(dossierId); + List files = fileStatusService.getActiveFiles(dossierId); dossierStats.setNumberOfFiles(files.size()); dossierStats.setNumberOfSoftDeletedFiles(fileStatusService.countSoftDeletedFiles(dossierId)); - dossierStats.setNumberOfPages(files.stream().mapToInt(FileEntity::getNumberOfPages).sum()); + dossierStats.setNumberOfPages(files.stream().mapToInt(FileModel::getNumberOfPages).sum()); dossierStats.setNumberOfExcludedPages(files.stream().mapToInt(f -> f.getExcludedPages().size()).sum()); - files.stream().filter(FileEntity::isHasRedactions).findAny().ifPresent( + files.stream().filter(FileModel::isHasRedactions).findAny().ifPresent( (v) -> dossierStats.setHasRedactionsFilePresent(true) ); - files.stream().filter(FileEntity::isHasHints).filter(f -> !f.isHasRedactions()).findAny().ifPresent( + files.stream().filter(FileModel::isHasHints).filter(f -> !f.isHasRedactions()).findAny().ifPresent( (v) -> dossierStats.setHasHintsNoRedactionsFilePresent(true) ); - files.stream().filter(FileEntity::isHasSuggestions).findAny().ifPresent( + files.stream().filter(FileModel::isHasSuggestions).findAny().ifPresent( (v) -> dossierStats.setHasSuggestionsFilePresent(true) ); - files.stream().filter(FileEntity::isHasUpdates).findAny().ifPresent( + files.stream().filter(FileModel::isHasUpdates).findAny().ifPresent( (v) -> dossierStats.setHasUpdatesFilePresent(true) ); files.stream().filter(f -> !f.isHasRedactions()) @@ -80,10 +86,10 @@ public class DossierStatsService { .findAny().ifPresent( (v) -> dossierStats.setHasNoFlagsFilePresent(true) ); - var fileCountPerProcessingStatus = files.stream().collect(Collectors.toMap(FileEntity::getProcessingStatus, e -> 1, Math::addExact)); + var fileCountPerProcessingStatus = files.stream().collect(Collectors.toMap(FileModel::getProcessingStatus, e -> 1, Math::addExact)); dossierStats.setFileCountPerProcessingStatus(fileCountPerProcessingStatus); - var fileCountPerWorkflowStatus = files.stream().collect(Collectors.toMap(FileEntity::getWorkflowStatus, e -> 1, Math::addExact)); + var fileCountPerWorkflowStatus = files.stream().collect(Collectors.toMap(FileModel::getWorkflowStatus, e -> 1, Math::addExact)); dossierStats.setFileCountPerWorkflowStatus(fileCountPerWorkflowStatus); return dossierStats; } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 769021f07..e971a44ad 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; import java.util.Arrays; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import org.springframework.stereotype.Service; import com.google.common.hash.HashFunction; @@ -97,7 +98,7 @@ public class FileService { } - private FileEntity retrieveStatus(String fileId) { + private FileModel retrieveStatus(String fileId) { try { return fileStatusService.getStatus(fileId); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index ebfd1dc97..b35b2758d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -1,16 +1,5 @@ package com.iqser.red.service.peristence.v1.server.service; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import javax.transaction.Transactional; - -import org.apache.commons.lang3.StringUtils; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.stereotype.Service; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; @@ -20,28 +9,37 @@ import com.iqser.red.service.peristence.v1.server.controller.RulesController; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; import com.iqser.red.service.peristence.v1.server.model.image.ImageServiceRequest; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; +import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.InternalServerErrorException; import com.iqser.red.service.persistence.management.v1.processor.exception.UserNotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.AddRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.CommentPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ForceRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ImageRecategorizationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.LegalBasisChangePersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.RemoveRedactionPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.ResizeRedactionPersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.redaction.v1.model.MessageType; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @Slf4j @Service @@ -62,24 +60,43 @@ public class FileStatusService { private final RemoveRedactionPersistenceService removeRedactionPersistenceService; private final AddRedactionPersistenceService addRedactionPersistenceService; private final ResizeRedactionPersistenceService resizeRedactionPersistenceService; + private final FileAttributeConfigPersistenceService fileAttributeConfigPersistenceService; private final FileManagementServiceSettings settings; + private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; + @Transactional + public List getAllRelevantStatusesForReanalysisScheduler() { + var fileEntities = fileStatusPersistenceService.getAllRelevantStatusesForReanalysisScheduler(); + var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList) + .stream() + .filter(FileModel::isAnalysisRequired) + .collect(Collectors.toList()); + } - public List getActiveFiles(String dossierId) { + @Transactional + public List getActiveFiles(String dossierId) { - return fileStatusPersistenceService.getActiveFiles(dossierId); + var fileEntities = fileStatusPersistenceService.getActiveFiles(dossierId); + var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); } - public List getDossierStatus(String dossierId) { + @Transactional + public List getDossierStatus(String dossierId) { - return fileStatusPersistenceService.getStatusesForDossier(dossierId); + var fileEntities = new ArrayList<>(fileStatusPersistenceService.getStatusesForDossier(dossierId)); + var convertedList = convert(fileEntities, FileModel.class, new FileModelMapper()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convertedList); } - public FileEntity getStatus(String fileId) { - - return fileStatusPersistenceService.getStatus(fileId); + @Transactional + public FileModel getStatus(String fileId) { + var fileEntity = fileStatusPersistenceService.getStatus(fileId); + var converted = convert(fileEntity, FileModel.class, new FileModelMapper()); + return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(converted); } @@ -169,7 +186,6 @@ public class FileStatusService { @Transactional - @SuppressWarnings("PMD") protected void addToAnalysisQueue(String dossierId, String fileId, int priority, Set sectionsToReanalyse) { var dossier = dossierPersistenceService.getAndValidateDossier(dossierId); @@ -205,7 +221,7 @@ public class FileStatusService { .manualRedactions(manualRedactionProviderService.getManualRedactions(fileId)) .dossierTemplateId(dossier.getDossierTemplateId()) .lastProcessed(fileStatus.getLastProcessed()) - .fileAttributes(convert(fileStatus.getFileAttributes())) + .fileAttributes(convertAttributes(fileStatus.getFileAttributes(), dossier.getDossierTemplateId())) .excludedPages(fileStatus.getExcludedPages()) .build(); @@ -341,6 +357,12 @@ public class FileStatusService { } + public void setStatusNerAnalyzing(String fileId) { + + fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING); + } + + public void setStatusOcrProcessing(String dossierId, String fileId) { FileEntity fileStatus = fileStatusPersistenceService.getStatus(fileId); @@ -355,12 +377,6 @@ public class FileStatusService { } - public void setStatusNerAnalyzing(String fileId) { - - fileStatusPersistenceService.updateProcessingStatus(fileId, ProcessingStatus.NER_ANALYZING); - } - - private void addToImageQueue(String dossierId, String fileId) { setStatusImageAnalyzing(fileId); @@ -403,12 +419,6 @@ public class FileStatusService { } - public List getAllStatuses() { - - return fileStatusPersistenceService.getAllStatuses(); - } - - public void wipeFileData(String dossierId, String fileId) { OffsetDateTime now = OffsetDateTime.now(); @@ -454,16 +464,23 @@ public class FileStatusService { } - private List convert( - List fileAttributes) { + private List convertAttributes( + List fileAttributes, String dossierTemplateId) { List fileAttributeList = new ArrayList<>(); + var fileAttributeConfigs = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId); + for (FileAttributeEntity fileAttribute : fileAttributes) { + + var config = fileAttributeConfigs.stream() + .filter(fac -> fac.getId().equals(fileAttribute.getFileAttributeId().getFileAttributeConfigId())) + .findFirst().orElseThrow(() -> new InternalServerErrorException("File Attribute config not defined for attribute :" + fileAttribute.getFileAttributeId())); + com.iqser.red.service.redaction.v1.model.FileAttribute attribute = com.iqser.red.service.redaction.v1.model.FileAttribute.builder() - .id(fileAttribute.getFileAttributeConfig().getId()) - .label(fileAttribute.getFileAttributeConfig().getLabel()) - .placeholder(fileAttribute.getFileAttributeConfig().getPlaceholder()) + .id(fileAttribute.getFileAttributeId().getFileAttributeConfigId()) + .label(config.getLabel()) + .placeholder(config.getPlaceholder()) .value(fileAttribute.getValue()) .build(); fileAttributeList.add(attribute); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java index c3b77fbc7..c86b3bcca 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/ReanalysisRequiredStatusService.java @@ -47,6 +47,7 @@ public class ReanalysisRequiredStatusService { entry.setFullAnalysisRequired(analysisRequiredResult.isFullAnalysisRequired()); }); + return fileModels; } @@ -87,17 +88,17 @@ public class ReanalysisRequiredStatusService { Map dossierMap) { // enhance with dossierTemplateId + DossierEntity dossier; try { - DossierEntity dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId())); - fileStatus.setDossierTemplateId(dossier.getDossierTemplateId()); + dossier = dossierMap.computeIfAbsent(fileStatus.getDossierId(), k -> dossierPersistenceService.getAndValidateDossier(fileStatus.getDossierId())); } catch (DossierNotFoundException e) { log.info("Dossier {} was not found, thus analysis is not required", fileStatus.getDossierId()); return new AnalysisRequiredResult(false, false); } // get relevant versions - var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(fileStatus.getDossierTemplateId(), - k -> buildVersionData(fileStatus.getDossierTemplateId())); + var dossierTemplateVersions = dossierTemplateVersionMap.computeIfAbsent(dossier.getDossierTemplateId(), + k -> buildVersionData(dossier.getDossierTemplateId())); var dossierDictionaryVersion = dossierVersionMap.computeIfAbsent(fileStatus.getDossierId(), k -> getDossierVersionData(fileStatus.getDossierId())); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadProcessorService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadProcessorService.java index bc528517b..59039d256 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadProcessorService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadProcessorService.java @@ -35,7 +35,7 @@ public class DownloadProcessorService { DownloadStatusEntity downloadStatus = downloadStatusPersistenceService.getStatus(downloadJob.getStorageId()); downloadStatusPersistenceService.updateStatus(downloadJob.getStorageId(), DownloadStatusValue.GENERATING); - var dossier = downloadStatus.getFiles().iterator().next().getDossier(); + var dossier = dossierService.getDossierById(downloadStatus.getFiles().iterator().next().getDossierId()); List filenameSortedFileIds = downloadStatus.getFiles() .stream() diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java index a243b5f22..ca2a1d350 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/AutomaticAnalysisScheduler.java @@ -4,10 +4,7 @@ import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfigu import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.peristence.v1.server.service.ReanalysisRequiredStatusService; import com.iqser.red.service.peristence.v1.server.utils.FileModelMapper; -import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.amqp.core.AmqpAdmin; @@ -17,6 +14,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.PostConstruct; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -30,7 +28,6 @@ public class AutomaticAnalysisScheduler { @Value("${persistence-service.automaticAnalysis.pageFactor:500}") private int pageFactor; private final FileStatusService fileStatusService; - private final ReanalysisRequiredStatusService reanalysisRequiredStatusService; private final AmqpAdmin amqpAdmin; @@ -53,7 +50,7 @@ public class AutomaticAnalysisScheduler { var consumerCount = redactionQueueInfo.getConsumerCount(); if (redactionQueueInfo.getMessageCount() <= consumerCount) { // queue up 5 files - var allStatuses = getAllStatuses(); + var allStatuses = getAllRelevantStatuses(); var allStatusesIterator = allStatuses.iterator(); log.info("Files that require reanalysis: {}", allStatuses.size()); @@ -80,27 +77,9 @@ public class AutomaticAnalysisScheduler { } - private List getAllStatuses() { - - return reanalysisRequiredStatusService.enhanceFileStatusWithAnalysisRequirements(convert(fileStatusService.getAllStatuses() - .stream() - .filter(f -> !f.getProcessingStatus().equals(ProcessingStatus.DELETED)) - .filter(f -> !f.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) - .filter(f -> !f.isExcludedFromAutomaticAnalysis()) - .filter(f -> !isProcessing(f)) - .filter(f -> f.getDossier().getSoftDeletedTime() == null && f.getDossier().getHardDeletedTime() == null && f.getDossier().getArchivedTime() == null) - .collect(Collectors.toList()), FileModel.class, new FileModelMapper())) - .stream() - .filter(FileModel::isAnalysisRequired) - .collect(Collectors.toList()); + private List getAllRelevantStatuses() { + return fileStatusService.getAllRelevantStatusesForReanalysisScheduler(); } - private boolean isProcessing(FileEntity file) { - - return !file.getProcessingStatus().equals(ProcessingStatus.PROCESSED) && !file.getProcessingStatus() - .equals(ProcessingStatus.UNPROCESSED) && !file.getProcessingStatus() - .equals(ProcessingStatus.DELETED) && !file.getProcessingStatus().equals(ProcessingStatus.ERROR); - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java index 54c433bb9..4f3937bb6 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/scheduler/DeletedFilesCleanupService.java @@ -40,13 +40,13 @@ public class DeletedFilesCleanupService { log.info("Hard deleted dossier with dossier id {} ", dossierEntity.getId()); } } else { - var fileEntities = fileStatusService.getDossierStatus(dossierEntity.getId()); - for (FileEntity fileEntity : fileEntities) { - if (fileEntity.getHardDeletedTime() == null && fileEntity.getDeleted() != null && fileEntity.getDeleted() + var files = fileStatusService.getDossierStatus(dossierEntity.getId()); + for (var file : files) { + if (file.getHardDeletedTime() == null && file.getDeleted() != null && file.getDeleted() .isBefore(now.minusHours(settings.getSoftDeleteCleanupTime()))) { - fileService.hardDeleteFile(dossierEntity.getId(), fileEntity.getId()); - fileStatusService.setFileStatusHardDeleted(fileEntity.getId()); - log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), fileEntity.getId()); + fileService.hardDeleteFile(dossierEntity.getId(), file.getId()); + fileStatusService.setFileStatusHardDeleted(file.getId()); + log.info("Hard deleted file with dossier id {} and file id {}", dossierEntity.getId(), file.getId()); } } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileModelMapper.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileModelMapper.java index 1652cd558..fa19e44f4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileModelMapper.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/utils/FileModelMapper.java @@ -14,8 +14,5 @@ public class FileModelMapper implements BiConsumer { fileModel.getFileAttributes().put(fa.getFileAttributeId().getFileAttributeConfigId(), fa.getValue()) ); - // Without this dossierTemplateId is not set for soft-deleted documents calling get softdeleted with curl. - // however in the unittest it always null without it. Strange.... - fileModel.setDossierTemplateId(fileEntity.getDossier().getDossierTemplateId()); } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml index e353f0592..931c54885 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml @@ -17,11 +17,23 @@ spring: username: ${PSQL_USERNAME:redaction} password: ${PSQL_PASSWORD:redaction} platform: org.hibernate.dialect.PostgreSQL95Dialect + hikari: + data-source-properties: + cachePrepStmts: true + prepStmtCacheSize: 1000 + prepStmtCacheSqlLimit: 2048 jpa: database-platform: org.hibernate.dialect.PostgreSQL95Dialect hibernate: ddl-auto: none naming-strategy: org.hibernate.cfg.ImprovedNamingStrategy + properties: + hibernate: + jdbc: + batch_size: 50 + order_inserts: true + order_updates: true + profiles: active: kubernetes diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/7-json-column-mapping.changelog.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/7-json-column-mapping.changelog.yaml new file mode 100644 index 000000000..843364d94 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/7-json-column-mapping.changelog.yaml @@ -0,0 +1,47 @@ +databaseChangeLog: + - changeSet: + id: add-file-excluded-pages-column + author: timo + changes: + - addColumn: + columns: + - column: + name: excluded_pages + type: TEXT + tableName: file + - changeSet: + id: add-dossier-download-file-types-members-approvers + author: timo + changes: + - addColumn: + columns: + - column: + name: members + type: TEXT + - column: + name: approvers + type: TEXT + - column: + name: download_file_types + type: TEXT + tableName: dossier + - changeSet: + id: add-dossier-template-download-file-types-members-approvers + author: timo + changes: + - addColumn: + columns: + - column: + name: download_file_types + type: TEXT + tableName: dossier_template + - changeSet: + id: add-download-status-download-file-types-members-approvers + author: timo + changes: + - addColumn: + columns: + - column: + name: download_file_types + type: TEXT + tableName: download_status diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index be1ef5a70..dfd54a7ee 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -11,3 +11,7 @@ databaseChangeLog: file: db/changelog/5-excluded-from-automatic-analysis-file-column.changelog.yaml - include: file: db/changelog/6-dossier-status-table.changelog.yaml + - include: + file: db/changelog/7-json-column-mapping.changelog.yaml + - include: + file: db/changelog/sql/7.1-set-json-fields.sql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/7.1-set-json-fields.sql b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/7.1-set-json-fields.sql new file mode 100644 index 000000000..d3e059786 --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/7.1-set-json-fields.sql @@ -0,0 +1,27 @@ +update file set excluded_pages = + (select '['||STRING_AGG(excluded_pages || '', ', ')||']' from file_entity_excluded_pages fe where fe.file_entity_id = file.id group by fe.file_entity_id); + +update dossier set members = + (select '['||STRING_AGG( '"'||member_ids || '"', ', ')||']' + from dossier_entity_member_ids dem where dem.dossier_entity_id = dossier.id + group by dem.dossier_entity_id); + +update dossier set approvers = + (select '['||STRING_AGG( '"'||approver_ids || '"', ', ')||']' + from dossier_entity_approver_ids dem where dem.dossier_entity_id = dossier.id + group by dem.dossier_entity_id); + +update dossier set download_file_types = + (select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']' + from dossier_entity_download_file_types dem where dem.dossier_entity_id = dossier.id + group by dem.dossier_entity_id); + +update dossier_template set download_file_types = + (select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']' + from dossier_template_entity_download_file_types dem where dem.dossier_template_entity_id = dossier_template.id + group by dem.dossier_template_entity_id); + +update download_status set download_file_types = + (select '['||STRING_AGG( '"'|| download_file_types || '"', ', ')||']' + from download_status_entity_download_file_types dem where dem.download_status_entity_storage_id = download_status.storage_id + group by dem.download_status_entity_storage_id); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java index 9ddcb8fd1..2a05257b9 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierStatsTest.java @@ -50,7 +50,7 @@ public class DossierStatsTest extends AbstractPersistenceServerServiceTest { var file1 = fileTesterAndProvider.testAndProvideFile(dossier1, "file1"); var file2 = fileTesterAndProvider.testAndProvideFile(dossier1, "file2"); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(2); + assertThat(fileClient.getDossierStatus(dossier1.getId()).size()).isEqualTo(2); // alter file 1 var loadedFile1 = fileClient.getFileStatus(dossier1.getId(), file1.getId()); assertThat(loadedFile1.isHasRedactions()).isFalse(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index 4abba0198..6922e420d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -81,7 +81,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var file = fileTesterAndProvider.testAndProvideFile(dossier, filename); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); assertThat(loadedFile.getFilename()).isEqualTo(filename); @@ -118,7 +118,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(changes.iterator().next().isFileChanges()).isTrue(); assertThat(changes.iterator().next().isDossierChanges()).isTrue(); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); var loadedFile = fileClient.getFileStatus(dossier.getId(), file.getId()); @@ -219,7 +219,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var softDeletedFiles = fileClient.getSoftDeletedDossierStatus(dossier.getId()); assertThat(softDeletedFiles.size()).isEqualTo(1); - assertThat(softDeletedFiles.get(0).getDossierTemplateId()).isNotNull(); var activeFiles = fileClient.getDossierStatus(dossier.getId()); assertThat(activeFiles.size()).isEqualTo(0); @@ -255,7 +254,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); String typeId = type.getId(); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), "test.pdf", dossier.getId(), "1"); JSONPrimitive uploadResult = uploadClient.upload(upload); @@ -344,7 +343,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); String typeId = type.getId(); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); var addRedaction = manualRedactionClient.addAddRedaction(dossierId, fileId, AddRedactionRequest.builder().addToDictionary(true) .addToDossierDictionary(false).comment("comment").status(AnnotationStatus.REQUESTED).typeId(typeId).user("user").reason("1").value("test").legalBasis("1").build()); @@ -380,9 +379,8 @@ public class FileTest extends AbstractPersistenceServerServiceTest { String fileId = file.getId(); var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual"); - String typeId = type.getId(); - assertThat(fileClient.getAllStatuses().size()).isEqualTo(1); + assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1); fileClient.excludePages(dossierId, fileId, Collections.singleton(1)); assertThat(fileClient.getFileStatus(dossierId, fileId).getExcludedPages().size()).isEqualTo(1); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java index 0fa67cd63..83b775601 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/MigrationTest.java @@ -4,7 +4,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -285,19 +288,24 @@ public class MigrationTest extends AbstractPersistenceServerServiceTest { } }); - var files = fileClient.getAllStatuses(); - files.forEach(file -> { - var manualRedactions = manualRedactionClient.getManualRedactions(file.getDossierId(), file.getId()); - if (file.getId().equals("a7985c094cac62118c9b350461902f95")) { - assertThat(manualRedactions.getEntriesToAdd().size()).isEqualTo(1); - assertThat(manualRedactions.getForceRedactions().size()).isEqualTo(1); - assertThat(manualRedactions.getImageRecategorization().size()).isEqualTo(0); - assertThat(manualRedactions.getLegalBasisChanges().size()).isEqualTo(1); - assertThat(manualRedactions.getIdsToRemove().size()).isEqualTo(1); - assertThat(manualRedactions.getComments().size()).isEqualTo(1); - } + List allFiles = new ArrayList<>(); + dossiers.forEach(dossier ->{ + var files = fileClient.getDossierStatus(dossier.getId()); + files.forEach(file -> { + var manualRedactions = manualRedactionClient.getManualRedactions(file.getDossierId(), file.getId()); + if (file.getId().equals("a7985c094cac62118c9b350461902f95")) { + assertThat(manualRedactions.getEntriesToAdd().size()).isEqualTo(1); + assertThat(manualRedactions.getForceRedactions().size()).isEqualTo(1); + assertThat(manualRedactions.getImageRecategorization().size()).isEqualTo(0); + assertThat(manualRedactions.getLegalBasisChanges().size()).isEqualTo(1); + assertThat(manualRedactions.getIdsToRemove().size()).isEqualTo(1); + assertThat(manualRedactions.getComments().size()).isEqualTo(1); + } + }); + allFiles.addAll(files); }); - assertThat(files.size()).isEqualTo(33); + assertThat(allFiles.size()).isEqualTo(33); + var digitalSignature = digitalSignatureClient.getDigitalSignature(); assertThat(digitalSignature.getCertificateName()).isNotEmpty(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java new file mode 100644 index 000000000..bd48c04ec --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java @@ -0,0 +1,221 @@ +package com.iqser.red.service.peristence.v1.server.integration.tests.performance; + +import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient; +import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatsClient; +import com.iqser.red.service.peristence.v1.server.integration.client.FileClient; +import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.*; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.DossierStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileAttributeType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.util.Sets; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + + +@Slf4j +public class FilePerformanceTest extends AbstractPersistenceServerServiceTest { + + private final static Integer DOSSIER_COUNT = 1; + private final static Integer FILE_PER_DOSSIER_COUNT = 100; + private final static Integer TYPE_COUNT = 1; + private final static Integer TYPE_ENTRY_COUNT = 500; + private final static Integer FILE_ATTRIBUTE_COUNT = 15; + + + @Autowired + private DossierClient dossierClient; + + @Autowired + private FileClient fileClient; + + @Autowired + private DossierStatsClient dossierStatsClient; + + @Before + public void setupTest() { + + long start = System.currentTimeMillis(); + + DossierTemplateEntity dte = new DossierTemplateEntity(); + dte.setId(UUID.randomUUID().toString()); + dte.setName("Test"); + dte.setDownloadFileTypes(Sets.newTreeSet(DownloadFileType.ANNOTATED, DownloadFileType.REDACTED, DownloadFileType.ORIGINAL, DownloadFileType.PREVIEW, DownloadFileType.FLATTEN)); + dte.setCreatedBy("123"); + dte.setDescription("Lorem Ipsum"); + dte.setModifiedBy("123"); + + dte = dossierTemplateRepository.save(dte); + + List fces = new ArrayList<>(); + for (int i = 0; i < FILE_ATTRIBUTE_COUNT; i++) { + FileAttributeConfigEntity fce = new FileAttributeConfigEntity(); + fce.setId(UUID.randomUUID().toString()); + fce.setDossierTemplate(dte); + fce.setLabel("123"); + fce.setCsvColumnHeader("123"); + fce.setPrimaryAttribute(true); + fce.setFilterable(true); + fce.setPlaceholder("Lorem Ipsum"); + fce.setType(FileAttributeType.TEXT); + fce.setDossierTemplate(dte); + fces.add(fce); + + } + + fces = fileAttributeConfigRepository.saveAll(fces); + + log.info("Created File Attributes Config"); + + for (int i = 0; i < TYPE_COUNT; i++) { + TypeEntity te = new TypeEntity(); + te.setId(UUID.randomUUID().toString()); + te.setDossierTemplate(dte); + te.setDescription("Lorem Ipsum"); + te.setLabel("Lorem ipsum"); + te.setType("Type " + i); + te.setAddToDictionaryAction(true); + te.setCaseInsensitive(true); + te.setHexColor("hexColor"); + te.setRank(10 + i); + te.setVersion(1231); + te.setRecommendation(true); + te.setHint(false); + + te = typeRepository.save(te); + + List des = new ArrayList<>(); + for (int j = 0; j < TYPE_ENTRY_COUNT; j++) { + DictionaryEntryEntity de = new DictionaryEntryEntity(); + de.setType(te); + de.setDeleted(false); + de.setVersion(1); + de.setValue("Dictionary Entry" + i); + + des.add(de); + } + entryRepository.saveAll(des); + + log.info("Created Type: {}", te.getType()); + } + + log.info("Created Dossier Template Data ..."); + + + for (int i = 0; i < DOSSIER_COUNT; i++) { + + DossierEntity d = new DossierEntity(); + d.setId(UUID.randomUUID().toString()); + d.setDossierName("Dossier " + i); + d.setDossierTemplate(dte); + d.setDueDate(OffsetDateTime.now()); + d.setStatus(DossierStatus.ACTIVE); + d.setDescription("Lorem Ipsum"); + d.setWatermarkEnabled(true); + d.setStartDate(OffsetDateTime.now()); + d.setApproverIds(Sets.newTreeSet("1", "2", "3", "4", "5", "6", "7", "8", "9", "10")); + d.setMemberIds(Sets.newTreeSet("1", "2", "3", "4", "5", "6", "7", "8", "9", "10")); + d.setDownloadFileTypes(Sets.newTreeSet(DownloadFileType.ANNOTATED, DownloadFileType.REDACTED, DownloadFileType.ORIGINAL, DownloadFileType.PREVIEW, DownloadFileType.FLATTEN)); + + d = this.dossierRepository.save(d); + + + List files = new ArrayList<>(); + for (int j = 0; j < FILE_PER_DOSSIER_COUNT; j++) { + FileEntity f = new FileEntity(); + f.setId(UUID.randomUUID().toString()); + f.setAdded(OffsetDateTime.now()); + f.setAssignee("123"); + f.setApprovalDate(OffsetDateTime.now()); + f.setAnalysisDuration(123); + f.setDictionaryVersion(33); + f.setDossierId(d.getId()); + f.setDossierDictionaryVersion(22); + f.setExcluded(false); + f.setExcludedPages(getExcludedPages()); + f.setFilename("Lorem Ipsum"); + f.setHasAnnotationComments(false); + f.setHasRedactions(true); + f.setHasImages(true); + f.setHasHints(true); + f.setUploader("!2321"); + f.setWorkflowStatus(WorkflowStatus.APPROVED); + f.setProcessingStatus(ProcessingStatus.FULLREPROCESS); + f.setLastUpdated(OffsetDateTime.now()); + f.setLastUploaded(OffsetDateTime.now()); + f.setLastProcessed(OffsetDateTime.now()); + f.setLastFileAttributeChange(OffsetDateTime.now()); + f.setLastManualRedaction(OffsetDateTime.now()); + f.setDossierId(d.getId()); + files.add(f); + + } + + log.info("Created Dossier: {}", d.getDossierName()); + files = fileRepository.saveAll(files); + + ArrayList fileAttributes = new ArrayList<>(); + List finalFces = fces; + files.forEach(file -> finalFces.forEach(fce -> { + FileAttributeEntity fa = new FileAttributeEntity(); + fa.setFileAttributeId(new FileAttributeEntity.FileAttributeEntityId()); + fa.getFileAttributeId().setFileId(file.getId()); + fa.getFileAttributeId().setFileAttributeConfigId(fce.getId()); + fa.setValue("Lorem Ipsum ..."); + fileAttributes.add(fa); + })); + fileAttributesRepository.saveAll(fileAttributes); + } + + long end = System.currentTimeMillis(); + + log.info("Created Performance data in: {}ms", end - start); + } + + @Test + public void doPerformanceTest() { + + + long startAllDossiers = System.currentTimeMillis(); + var allDossiers = dossierClient.getAllDossiers(false, false); + log.info("Load all dossiers time: {}ms", System.currentTimeMillis() - startAllDossiers); + assertThat(allDossiers.size()).isEqualTo(DOSSIER_COUNT); + + long startDossierStats = System.currentTimeMillis(); + var dossierStats = dossierStatsClient.getDossierStats(allDossiers.stream().map(Dossier::getId).collect(Collectors.toSet())); + log.info("Load all dossiers stats time: {}ms", System.currentTimeMillis() - startDossierStats); + assertThat(dossierStats.size()).isEqualTo(DOSSIER_COUNT); + + for (var dossier : allDossiers) { + long startFilesTime = System.currentTimeMillis(); + var files = fileClient.getDossierStatus(dossier.getId()); + assertThat(files.size()).isEqualTo(FILE_PER_DOSSIER_COUNT); + log.info("Load files for dossier {} time: {}ms", dossier.getId(), System.currentTimeMillis() - startFilesTime); + } + + } + + private Set getExcludedPages() { + + Set pages = new HashSet<>(); + for (int i = 0; i < 100; i++) { + pages.add(i); + } + return pages; + } + + +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 69f7f4d97..bc071558b 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -48,7 +48,8 @@ import java.util.TimeZone; @EnableFeignClients(basePackageClasses = FileClient.class) @Import(AbstractPersistenceServerServiceTest.TestConfiguration.class) @ContextConfiguration(initializers = {AbstractPersistenceServerServiceTest.Initializer.class}) -@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, + properties = "spring-hibernate-query-utils.n-plus-one-queries-detection.error-level=INFO") public abstract class AbstractPersistenceServerServiceTest { @MockBean @@ -62,61 +63,63 @@ public abstract class AbstractPersistenceServerServiceTest { @MockBean protected PDFTronRedactionClient pdfTronRedactionClient; @Autowired - private StorageService storageService; + protected StorageService storageService; @Autowired - private DossierTemplateRepository dossierTemplateRepository; + protected DossierTemplateRepository dossierTemplateRepository; @Autowired - private DossierRepository dossierRepository; + protected DossierRepository dossierRepository; @Autowired protected FileRepository fileRepository; @Autowired - private ViewedPagesRepository viewedPagesRepository; + protected ViewedPagesRepository viewedPagesRepository; @Autowired - private NotificationRepository notificationRepository; + protected NotificationRepository notificationRepository; @Autowired - private AuditRepository auditRepository; + protected AuditRepository auditRepository; @Autowired - private TypeRepository typeRepository; + protected TypeRepository typeRepository; @Autowired - private ManualRedactionRepository manualRedactionRepository; + protected ManualRedactionRepository manualRedactionRepository; @Autowired - private ForceRedactionRepository forceRedactionRepository; + protected ForceRedactionRepository forceRedactionRepository; @Autowired - private RemoveRedactionRepository removeRedactionRepository; + protected RemoveRedactionRepository removeRedactionRepository; @Autowired - private LegalBasisChangeRepository legalBasisChangeRepository; + protected LegalBasisChangeRepository legalBasisChangeRepository; @Autowired - private ImageRecategorizationRepository imageRecategorizationRepository; + protected ImageRecategorizationRepository imageRecategorizationRepository; @Autowired - private WatermarkRepository watermarkRepository; + protected WatermarkRepository watermarkRepository; @Autowired - private SMTPRepository smtpRepository; + protected SMTPRepository smtpRepository; @Autowired - private RuleSetRepository ruleSetRepository; + protected RuleSetRepository ruleSetRepository; @Autowired - private LegalBasisMappingRepository legalBasisMappingRepository; + protected LegalBasisMappingRepository legalBasisMappingRepository; @Autowired - private FileAttributeConfigRepository fileAttributeConfigRepository; + protected FileAttributeConfigRepository fileAttributeConfigRepository; @Autowired - private FileAttributesGeneralConfigurationRepository fileAttributesGeneralConfigurationRepository; + protected FileAttributesGeneralConfigurationRepository fileAttributesGeneralConfigurationRepository; @Autowired - private ReportTemplateRepository reportTemplateRepository; + protected ReportTemplateRepository reportTemplateRepository; @Autowired - private DigitalSignatureRepository digitalSignatureRepository; + protected DigitalSignatureRepository digitalSignatureRepository; @Autowired - private FileAttributesRepository fileAttributesRepository; + protected FileAttributesRepository fileAttributesRepository; @Autowired - private DownloadStatusRepository downloadStatusRepository; + protected DownloadStatusRepository downloadStatusRepository; @Autowired - private DossierAttributeRepository dossierAttributeRepository; + protected DossierAttributeRepository dossierAttributeRepository; @Autowired - private DossierAttributeConfigRepository dossierAttributeConfigRepository; + protected DossierAttributeConfigRepository dossierAttributeConfigRepository; @Autowired - private NotificationPreferencesRepository notificationPreferencesRepository; + protected NotificationPreferencesRepository notificationPreferencesRepository; @Autowired private DossierStatusRepository dossierStatusRepository; @Autowired - private MetricsPrinterService metricsPrinterService; + protected MetricsPrinterService metricsPrinterService; + @Autowired + protected EntryRepository entryRepository; @Before public void setupOptimize() { @@ -162,6 +165,7 @@ public abstract class AbstractPersistenceServerServiceTest { @After public void afterTests() { + entryRepository.deleteAll(); dossierAttributeRepository.deleteAll(); dossierAttributeConfigRepository.deleteAll(); downloadStatusRepository.deleteAll(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java index ae2c759aa..341c0a236 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/MetricsPrinterService.java @@ -17,9 +17,16 @@ public class MetricsPrinterService { @Value("${server.port}") private int serverPort; + @Value("${management.endpoint.metrics.enabled}") + private boolean metricsEnabled; + @SneakyThrows public void printMetrics() { + if(!metricsEnabled){ + return; + } + RestTemplate rt = new RestTemplate(); var url = "http://127.0.0.1:" + serverPort + "/actuator/metrics"; diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml index 877e0801c..a7b3f4c60 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml @@ -5,13 +5,23 @@ spring: datasource: driverClassName: org.postgresql.Driver platform: org.hibernate.dialect.PostgreSQL95Dialect + hikari: + data-source-properties: + cachePrepStmts: true + prepStmtCacheSize: 1000 + prepStmtCacheSqlLimit: 2048 jpa: database-platform: org.hibernate.dialect.PostgreSQL95Dialect hibernate: ddl-auto: none naming-strategy: org.hibernate.cfg.ImprovedNamingStrategy generate-ddl: false - + properties: + hibernate: + jdbc: + batch_size: 50 + order_inserts: true + order_updates: true rabbitmq: host: ${RABBITMQ_HOST:localhost} @@ -59,10 +69,10 @@ persistence-service: metrics: persistence: - enabled: true + enabled: false management: endpoint: - metrics.enabled: true + metrics.enabled: false health.enabled: true endpoints.web.exposure.include: prometheus, health, metrics