RED-3942 - Improve clone performance
This commit is contained in:
Timo Bejan
parent
a2864b4e1c
commit
06c9162b0f
@ -100,6 +100,5 @@
|
||||
<artifactId>guava</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
@ -4,13 +4,10 @@ public interface BaseDictionaryEntry {
|
||||
|
||||
String getValue();
|
||||
|
||||
|
||||
long getVersion();
|
||||
|
||||
|
||||
boolean isDeleted();
|
||||
|
||||
|
||||
TypeEntity getType();
|
||||
String getTypeId();
|
||||
|
||||
}
|
||||
|
||||
@ -17,7 +17,7 @@ import javax.persistence.*;
|
||||
public class DictionaryEntryEntity implements BaseDictionaryEntry {
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private long entryId;
|
||||
@Column(length = 4000)
|
||||
private String value;
|
||||
@ -25,9 +25,8 @@ public class DictionaryEntryEntity implements BaseDictionaryEntry {
|
||||
private long version;
|
||||
@Column
|
||||
private boolean deleted;
|
||||
@JsonIgnore
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private TypeEntity type;
|
||||
@Column(name = "type_id")
|
||||
private String typeId;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -1,12 +1,6 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.FetchType;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.ManyToOne;
|
||||
import javax.persistence.Table;
|
||||
import javax.persistence.*;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
@ -24,7 +18,7 @@ import lombok.NoArgsConstructor;
|
||||
public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry {
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private long entryId;
|
||||
@Column(length = 4000)
|
||||
private String value;
|
||||
@ -32,10 +26,8 @@ public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry {
|
||||
private long version;
|
||||
@Column
|
||||
private boolean deleted;
|
||||
|
||||
@JsonIgnore
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private TypeEntity type;
|
||||
@Column( name = "type_id")
|
||||
private String typeId;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -1,20 +1,12 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.FetchType;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.ManyToOne;
|
||||
import javax.persistence.Table;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@Entity
|
||||
@ -24,7 +16,7 @@ import lombok.NoArgsConstructor;
|
||||
public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryEntry {
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private long entryId;
|
||||
@Column(length = 4000)
|
||||
private String value;
|
||||
@ -32,10 +24,8 @@ public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryE
|
||||
private long version;
|
||||
@Column
|
||||
private boolean deleted;
|
||||
|
||||
@JsonIgnore
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
private TypeEntity type;
|
||||
@Column(name = "type_id")
|
||||
private String typeId;
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
|
||||
@ -13,7 +16,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.Cl
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
|
||||
import com.iqser.red.storage.commons.service.StorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -25,9 +27,7 @@ import java.time.OffsetDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
|
||||
|
||||
@ -57,7 +57,6 @@ public class DossierTemplateCloneService {
|
||||
|
||||
DossierTemplateEntity clonedDossierTemplate = new DossierTemplateEntity();
|
||||
|
||||
long start=System.currentTimeMillis();
|
||||
dossierTemplateRepository.findById(dossierTemplateId).ifPresentOrElse((dossierTemplate) -> {
|
||||
|
||||
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
|
||||
@ -72,7 +71,6 @@ public class DossierTemplateCloneService {
|
||||
clonedDossierTemplate.setValidTo(cloneDossierTemplateRequest.getValidTo() == null ? dossierTemplate.getValidTo() : cloneDossierTemplateRequest.getValidTo());
|
||||
clonedDossierTemplate.setSoftDeleteTime(dossierTemplate.getSoftDeleteTime());
|
||||
clonedDossierTemplate.setDownloadFileTypes(cloneDossierTemplateRequest.getDownloadFileTypes() == null || cloneDossierTemplateRequest.getDownloadFileTypes().isEmpty() ? dossierTemplate.getDownloadFileTypes() : cloneDossierTemplateRequest.getDownloadFileTypes());
|
||||
|
||||
//set rules
|
||||
cloneRules(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
//set legal basis
|
||||
@ -83,21 +81,17 @@ public class DossierTemplateCloneService {
|
||||
dossierTemplateRepository.save(clonedDossierTemplate);
|
||||
|
||||
//set dictionaries
|
||||
long t1 = System.currentTimeMillis();
|
||||
clonedDossierTemplate.setDossierTypes(cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId()));
|
||||
long t2 = System.currentTimeMillis();
|
||||
log.warn("Time to clone entries: {}", (t2 - t1));
|
||||
cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
|
||||
//set dossier attributes
|
||||
cloneDossierAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
|
||||
//set file attributes
|
||||
cloneFileAttributesGeneralConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
|
||||
clonedDossierTemplate.setFileAttributeConfigs(cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId()));
|
||||
cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
|
||||
//set report templates
|
||||
clonedDossierTemplate.setReportTemplates(cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId()));
|
||||
cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId());
|
||||
|
||||
//set watermarks
|
||||
cloneWatermark(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
@ -108,14 +102,10 @@ public class DossierTemplateCloneService {
|
||||
//set dossier status
|
||||
cloneDossierStates(dossierTemplate.getId(), clonedDossierTemplate.getId());
|
||||
|
||||
dossierTemplateRepository.save(clonedDossierTemplate);
|
||||
|
||||
}, () -> {
|
||||
throw new NotFoundException(String.format(dossierTemplatePersistenceService.DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE, dossierTemplateId));
|
||||
});
|
||||
long end = System.currentTimeMillis();
|
||||
System.out.println("END TIME: "+(end-start));
|
||||
log.warn("End Time {}",end-start);
|
||||
|
||||
return clonedDossierTemplate;
|
||||
}
|
||||
|
||||
@ -132,23 +122,13 @@ public class DossierTemplateCloneService {
|
||||
}
|
||||
|
||||
|
||||
private List<TypeEntity> cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
|
||||
List<TypeEntity> clonedTypes = new ArrayList<>();
|
||||
var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
|
||||
for (TypeEntity t : types) {
|
||||
TypeEntity te = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped());
|
||||
te.setDossierTemplateId(clonedDossierTemplateId);
|
||||
clonedTypes.add(te);
|
||||
// for (DictionaryEntryType det : DictionaryEntryType.values()) {
|
||||
// var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null);
|
||||
// Set<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet());
|
||||
// entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det);
|
||||
// }
|
||||
|
||||
entryPersistenceService.cloneEntries(dossierTemplateId,clonedDossierTemplateId);
|
||||
var type = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped());
|
||||
entryPersistenceService.cloneEntries(t.getId(), type.getId());
|
||||
}
|
||||
return clonedTypes;
|
||||
}
|
||||
|
||||
|
||||
@ -179,13 +159,13 @@ public class DossierTemplateCloneService {
|
||||
.filenameMappingColumnHeaderName(fileAttributesGeneralConfig.getFilenameMappingColumnHeaderName())
|
||||
.build();
|
||||
fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(clonedDossierTemplateId, fagc);
|
||||
}catch (NotFoundException e){
|
||||
} catch (NotFoundException e) {
|
||||
log.debug("No file attribute config found for cloning {}", dossierTemplateId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private List<FileAttributeConfigEntity> cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
private void cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
|
||||
List<FileAttributeConfigEntity> facList = new ArrayList<>();
|
||||
var fileAttributesConfig = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
|
||||
@ -203,14 +183,12 @@ public class DossierTemplateCloneService {
|
||||
facList.add(fac);
|
||||
});
|
||||
fileAttributeConfigPersistenceService.setFileAttributesConfig(clonedDossierTemplateId, facList);
|
||||
|
||||
return fileAttributeConfigPersistenceService.getFileAttributes(clonedDossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
private List<ReportTemplateEntity> cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) {
|
||||
private void cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) {
|
||||
|
||||
var reportTemplates = dossierTemplate.getReportTemplates();
|
||||
var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId());
|
||||
for (ReportTemplateEntity rte : reportTemplates) {
|
||||
var storedReportTemplate = storageService.getObject(rte.getStorageId());
|
||||
String storageId = clonedDossierTemplateId + "/" + rte.getFileName();
|
||||
@ -222,7 +200,6 @@ public class DossierTemplateCloneService {
|
||||
}
|
||||
reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport());
|
||||
}
|
||||
return reportTemplatePersistenceService.findByDossierTemplateId(clonedDossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
@ -240,7 +217,7 @@ public class DossierTemplateCloneService {
|
||||
.orientation(watermark.getOrientation())
|
||||
.build();
|
||||
watermarkService.saveWatermark(clonedDossierTemplateId, we);
|
||||
}catch (NotFoundException e){
|
||||
} catch (NotFoundException e) {
|
||||
log.debug("No watermark config found for cloning {}", dossierTemplateId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -27,6 +28,7 @@ public class EntryPersistenceService {
|
||||
private final TypeRepository typeRepository;
|
||||
private final FalsePositiveEntryRepository falsePositiveEntryRepository;
|
||||
private final FalseRecommendationEntryRepository falseRecommendationEntryRepository;
|
||||
private final JDBCWriteUtils jdbcWriteUtils;
|
||||
|
||||
@Transactional
|
||||
public void deleteEntries(String typeId, List<String> values, long version,
|
||||
@ -102,12 +104,12 @@ public class EntryPersistenceService {
|
||||
DictionaryEntryEntity entry = new DictionaryEntryEntity();
|
||||
entry.setVersion(version);
|
||||
entry.setValue(e);
|
||||
entry.setType(type);
|
||||
entry.setTypeId(type.getId());
|
||||
return entry;
|
||||
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
entryRepository.saveAll(entryEntities);
|
||||
jdbcWriteUtils.saveBatch(entryEntities);
|
||||
break;
|
||||
}
|
||||
case FALSE_POSITIVE: {
|
||||
@ -118,12 +120,11 @@ public class EntryPersistenceService {
|
||||
DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity();
|
||||
entry.setVersion(version);
|
||||
entry.setValue(e);
|
||||
entry.setType(type);
|
||||
entry.setTypeId(type.getId());
|
||||
return entry;
|
||||
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
falsePositiveEntryRepository.saveAll(entryEntities);
|
||||
jdbcWriteUtils.saveBatch(entryEntities);
|
||||
break;
|
||||
}
|
||||
case FALSE_RECOMMENDATION: {
|
||||
@ -134,20 +135,19 @@ public class EntryPersistenceService {
|
||||
DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity();
|
||||
entry.setVersion(version);
|
||||
entry.setValue(e);
|
||||
entry.setType(type);
|
||||
entry.setTypeId(type.getId());
|
||||
return entry;
|
||||
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
falseRecommendationEntryRepository.saveAll(entryEntities);
|
||||
jdbcWriteUtils.saveBatch(entryEntities);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void cloneEntries(String dossierTemplateId, String clonedDossierTemplateId) {
|
||||
entryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
|
||||
// falseRecommendationEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
|
||||
// falsePositiveEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
|
||||
public void cloneEntries(String originalTypeId, String newTypeId){
|
||||
entryRepository.cloneEntries(originalTypeId, newTypeId);
|
||||
falseRecommendationEntryRepository.cloneEntries(originalTypeId, newTypeId);
|
||||
falsePositiveEntryRepository.cloneEntries(originalTypeId, newTypeId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,24 +12,24 @@ import java.util.Set;
|
||||
public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Long> {
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id =:typeId and e.value in :values")
|
||||
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
|
||||
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
|
||||
@Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
|
||||
void updateVersionWhereTypeId(long version, String typeId);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId and e.deleted = false")
|
||||
@Query("update DictionaryEntryEntity e set e.typeId = :newTypeId, e.version = e.version + 1 where e.typeId =:typeId and e.deleted = false")
|
||||
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
|
||||
|
||||
List<DictionaryEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
|
||||
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
|
||||
void deleteAllEntriesForTypeId(String typeId, long version);
|
||||
|
||||
|
||||
@ -42,6 +42,6 @@ public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Lo
|
||||
@Modifying(flushAutomatically = true, clearAutomatically = true)
|
||||
@Transactional
|
||||
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " +
|
||||
" select value, 1, false, :targetTypeId from dictionary_entry where type_id = :originTypeId and deleted = false", nativeQuery = true)
|
||||
void cloneEntries(String targetTypeId, String originTypeId);
|
||||
" select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
|
||||
void cloneEntries(String originalTypeId, String newTypeId);
|
||||
}
|
||||
|
||||
@ -12,12 +12,12 @@ import java.util.Set;
|
||||
public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
|
||||
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
|
||||
void updateVersionWhereTypeId(long version, String typeId);
|
||||
|
||||
|
||||
@ -25,11 +25,17 @@ public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFa
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
|
||||
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
|
||||
void deleteAllEntriesForTypeId(String typeId, long version);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(value = "update dictionary_false_positive_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
|
||||
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
|
||||
|
||||
@Modifying(flushAutomatically = true, clearAutomatically = true)
|
||||
@Transactional
|
||||
@Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " +
|
||||
" select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
|
||||
void cloneEntries(String originalTypeId, String newTypeId);
|
||||
}
|
||||
|
||||
@ -1,26 +1,23 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
|
||||
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
|
||||
import javax.transaction.Transactional;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public interface FalseRecommendationEntryRepository extends JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
|
||||
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
|
||||
|
||||
|
||||
@Modifying
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
|
||||
void updateVersionWhereTypeId(long version, String typeId);
|
||||
|
||||
|
||||
@ -28,11 +25,17 @@ public interface FalseRecommendationEntryRepository extends JpaRepository<Dictio
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
|
||||
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
|
||||
void deleteAllEntriesForTypeId(String typeId, long version);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(value = "update dictionary_false_recommendation_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
|
||||
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
|
||||
|
||||
@Modifying(flushAutomatically = true, clearAutomatically = true)
|
||||
@Transactional
|
||||
@Query(value = "insert into dictionary_false_recommendation_entry (value, version, deleted, type_id) " +
|
||||
" select value, 1, false, :newTypeId from dictionary_false_recommendation_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
|
||||
void cloneEntries(String originalTypeId, String newTypeId);
|
||||
}
|
||||
|
||||
@ -39,7 +39,7 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
|
||||
@Query("select new com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e)) " +
|
||||
"from DossierTemplateEntity dt " +
|
||||
"left join TypeEntity t on t.dossierTemplateId = dt.id " +
|
||||
"left join DictionaryEntryEntity e on e.type.id = t.id " +
|
||||
"left join DictionaryEntryEntity e on e.typeId = t.id " +
|
||||
"where dt.id in :dossierTemplateIds and t.dossierId is null and (e.id is null or e.deleted = false) " +
|
||||
"group by dt.id, t.id, t.type, t.label ")
|
||||
List<DictionarySummaryResponse> findDictionarySummaryList(Set<String> dossierTemplateIds);
|
||||
|
||||
@ -0,0 +1,135 @@
|
||||
package com.iqser.red.service.persistence.management.v1.processor.utils.jdbc;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.EntityManager;
|
||||
import javax.persistence.Table;
|
||||
import java.lang.reflect.Field;
|
||||
import java.lang.reflect.Method;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.capitalize;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class JDBCWriteUtils {
|
||||
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
|
||||
private final String SQL_TEMPLATE = "INSERT INTO %s (%s) values (%s)";
|
||||
|
||||
private final Map<Class<?>, EntityMetadata> entityMetadataMap = new HashMap<>();
|
||||
|
||||
private final EntityManager entityManager;
|
||||
|
||||
public <T> void saveBatch(final List<T> entities) {
|
||||
if (entities.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
var metadata = getEntityMetadata(entities.iterator().next());
|
||||
|
||||
final int batchSize = 500;
|
||||
|
||||
for (int j = 0; j < entities.size(); j += batchSize) {
|
||||
|
||||
final List<T> batchList = entities.subList(j, Math.min(j + batchSize, entities.size()));
|
||||
|
||||
jdbcTemplate.batchUpdate(metadata.getSqlStatement(),
|
||||
new BatchPreparedStatementSetter() {
|
||||
@SneakyThrows
|
||||
@Override
|
||||
public void setValues(PreparedStatement ps, int i) {
|
||||
T entity = batchList.get(i);
|
||||
|
||||
int paramIndex = 1;
|
||||
for (var mapping : metadata.getFieldMethodMap().entrySet()) {
|
||||
ps.setObject(paramIndex++, mapping.getValue().invoke(entity));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getBatchSize() {
|
||||
return batchList.size();
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
entityManager.clear();
|
||||
}
|
||||
|
||||
private <T> EntityMetadata getEntityMetadata(T entity) {
|
||||
|
||||
var existingMetadata = entityMetadataMap.get(entity.getClass());
|
||||
if (existingMetadata != null) {
|
||||
return existingMetadata;
|
||||
}
|
||||
|
||||
var tableName = getTableName(entity);
|
||||
var args = getArgs(entity);
|
||||
var sql = String.format(SQL_TEMPLATE, tableName, String.join(", ", args.keySet()), args.keySet().stream().map(a -> "?").collect(Collectors.joining(", ")));
|
||||
|
||||
var metadata = new EntityMetadata(tableName, sql, args);
|
||||
entityMetadataMap.put(entity.getClass(), metadata);
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@SneakyThrows
|
||||
private <T> Map<String, Method> getArgs(T entity) {
|
||||
var fields = entity.getClass().getDeclaredFields();
|
||||
Map<String, Method> entityMethodMap = new LinkedHashMap<>();
|
||||
for (var field : fields) {
|
||||
var annotations = field.getDeclaredAnnotations();
|
||||
for (var annotation : annotations) {
|
||||
if (annotation.annotationType().equals(Column.class)) {
|
||||
var columnAnnotation = (Column) annotation;
|
||||
var name = StringUtils.isEmpty(columnAnnotation.name()) ? toSnakeCase(field.getName()) : columnAnnotation.name();
|
||||
entityMethodMap.put(name, entity.getClass().getMethod(getMethodName(field)));
|
||||
}
|
||||
}
|
||||
}
|
||||
return entityMethodMap;
|
||||
}
|
||||
|
||||
private String getMethodName(Field field) {
|
||||
var prefix = "get";
|
||||
if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) {
|
||||
prefix = "is";
|
||||
}
|
||||
return prefix + capitalize(field.getName());
|
||||
}
|
||||
|
||||
private String toSnakeCase(String name) {
|
||||
String ret = name.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2");
|
||||
return ret.toLowerCase();
|
||||
}
|
||||
|
||||
private <T> String getTableName(T entity) {
|
||||
var tableAnnot = entity.getClass().getDeclaredAnnotation(Table.class);
|
||||
return tableAnnot.name();
|
||||
}
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
public static class EntityMetadata {
|
||||
|
||||
private String tableName;
|
||||
private String sqlStatement;
|
||||
private Map<String, Method> fieldMethodMap;
|
||||
}
|
||||
}
|
||||
@ -77,9 +77,7 @@ public class DictionaryController implements DictionaryResource {
|
||||
|
||||
if (removeCurrent) {
|
||||
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
|
||||
long t1 = System.currentTimeMillis();
|
||||
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
|
||||
System.out.println("STUFF: {} "+(System.currentTimeMillis()-t1));
|
||||
} else {
|
||||
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@ spring:
|
||||
main:
|
||||
allow-circular-references: true # FIXME
|
||||
datasource:
|
||||
url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction}
|
||||
url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction}?cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true
|
||||
driverClassName: org.postgresql.Driver
|
||||
username: ${PSQL_USERNAME:redaction}
|
||||
password: ${PSQL_PASSWORD:redaction}
|
||||
|
||||
@ -71,3 +71,5 @@ databaseChangeLog:
|
||||
file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml
|
||||
- include:
|
||||
file: db/changelog/29-add-remove-digital-signatures-on-upload.changelog.yaml
|
||||
- include:
|
||||
file: db/changelog/sql/30-change-bigint-to-serial.sql
|
||||
|
||||
@ -0,0 +1,7 @@
|
||||
ALTER TABLE dictionary_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
|
||||
ALTER TABLE dictionary_false_recommendation_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
|
||||
ALTER TABLE dictionary_false_positive_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
|
||||
|
||||
SELECT setval(pg_get_serial_sequence('dictionary_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_entry));
|
||||
SELECT setval(pg_get_serial_sequence('dictionary_false_recommendation_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_recommendation_entry));
|
||||
SELECT setval(pg_get_serial_sequence('dictionary_false_positive_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_positive_entry));
|
||||
@ -8,12 +8,16 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers
|
||||
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
|
||||
@Slf4j
|
||||
public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest {
|
||||
|
||||
@Autowired
|
||||
@ -42,24 +46,38 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
|
||||
var tenKEntries = generateEntries(10000);
|
||||
|
||||
long t1 = System.currentTimeMillis();
|
||||
dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
|
||||
dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY);
|
||||
long t2 = System.currentTimeMillis();
|
||||
System.out.println("Time T1: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
|
||||
log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
|
||||
|
||||
|
||||
t1 = System.currentTimeMillis();
|
||||
dictionaryClient.addEntries(type.getTypeId(),tenKEntries , true, false, DictionaryEntryType.ENTRY);
|
||||
dictionaryClient.addEntries(type.getTypeId(), tenKEntries, true, false, DictionaryEntryType.ENTRY);
|
||||
t2 = System.currentTimeMillis();
|
||||
System.out.println("Time T2: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
|
||||
log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
|
||||
|
||||
|
||||
t1 = System.currentTimeMillis();
|
||||
dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
|
||||
dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY);
|
||||
t2 = System.currentTimeMillis();
|
||||
System.out.println("Time T3: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
|
||||
log.info("Update Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
|
||||
|
||||
|
||||
dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
|
||||
dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_RECOMMENDATION);
|
||||
dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_POSITIVE);
|
||||
|
||||
t1 = System.currentTimeMillis();
|
||||
var cloned = dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
|
||||
t2 = System.currentTimeMillis();
|
||||
log.info("Clone Time: {}", (t2 - t1));
|
||||
|
||||
var types = dictionaryClient.getAllTypesForDossierTemplate(cloned.getId(), false);
|
||||
|
||||
assertThat(types.size()).isEqualTo(1);
|
||||
for (var clonedType : types) {
|
||||
var found = entryRepository.findByTypeIdAndVersionGreaterThan(clonedType.getTypeId(), 0);
|
||||
assertThat(found.size()).isEqualTo(5000);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -100,7 +100,7 @@ public class FilePerformanceTest extends AbstractPersistenceServerServiceTest {
|
||||
List<DictionaryEntryEntity> des = new ArrayList<>();
|
||||
for (int j = 0; j < TYPE_ENTRY_COUNT; j++) {
|
||||
DictionaryEntryEntity de = new DictionaryEntryEntity();
|
||||
de.setType(te);
|
||||
de.setTypeId(te.getId());
|
||||
de.setDeleted(false);
|
||||
de.setVersion(1);
|
||||
de.setValue("Dictionary Entry" +i+"/"+ j);
|
||||
|
||||
@ -24,7 +24,7 @@ public class FileSystemBackArchiverTest {
|
||||
|
||||
SplittableRandom sr = new SplittableRandom();
|
||||
|
||||
var data = sr.doubles().limit(1024 * 1024 * 64).toArray();
|
||||
var data = sr.doubles().limit(1024 * 1024).toArray();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
log.info("At entry: {}, using {}MB of memory", i, (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024));
|
||||
|
||||
|
||||
@ -77,3 +77,5 @@ management:
|
||||
metrics.enabled: false
|
||||
health.enabled: true
|
||||
endpoints.web.exposure.include: prometheus, health, metrics
|
||||
|
||||
logging.level.root: info
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user