diff --git a/persistence-service-v1/persistence-service-processor-v1/pom.xml b/persistence-service-v1/persistence-service-processor-v1/pom.xml index bcca653f1..907c93920 100644 --- a/persistence-service-v1/persistence-service-processor-v1/pom.xml +++ b/persistence-service-v1/persistence-service-processor-v1/pom.xml @@ -100,6 +100,5 @@ guava compile - diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java index d7b1326be..e61260539 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java @@ -4,13 +4,10 @@ public interface BaseDictionaryEntry { String getValue(); - long getVersion(); - boolean isDeleted(); - - TypeEntity getType(); + String getTypeId(); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java index 10e2e6353..92c5d1d26 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java @@ -17,7 +17,7 @@ import javax.persistence.*; public class DictionaryEntryEntity implements BaseDictionaryEntry { @Id - @GeneratedValue + @GeneratedValue(strategy = GenerationType.IDENTITY) private long entryId; @Column(length = 4000) private String value; @@ -25,9 +25,8 @@ public class DictionaryEntryEntity implements BaseDictionaryEntry { private long version; @Column private boolean deleted; - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - private TypeEntity type; + @Column(name = "type_id") + private String typeId; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java index faf427b30..8d3b626b7 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java @@ -1,12 +1,6 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.ManyToOne; -import javax.persistence.Table; +import javax.persistence.*; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -24,7 +18,7 @@ import lombok.NoArgsConstructor; public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry { @Id - @GeneratedValue + @GeneratedValue(strategy = GenerationType.IDENTITY) private long entryId; @Column(length = 4000) private String value; @@ -32,10 +26,8 @@ public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry { private long version; @Column private boolean deleted; - - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - private TypeEntity type; + @Column( name = "type_id") + private String typeId; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java index f1d7a760d..d5020f428 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java @@ -1,20 +1,12 @@ package com.iqser.red.service.persistence.management.v1.processor.entity.configuration; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.ManyToOne; -import javax.persistence.Table; - -import com.fasterxml.jackson.annotation.JsonIgnore; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import javax.persistence.*; + @Data @Builder @Entity @@ -24,7 +16,7 @@ import lombok.NoArgsConstructor; public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryEntry { @Id - @GeneratedValue + @GeneratedValue(strategy = GenerationType.IDENTITY) private long entryId; @Column(length = 4000) private String value; @@ -32,10 +24,8 @@ public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryE private long version; @Column private boolean deleted; - - @JsonIgnore - @ManyToOne(fetch = FetchType.LAZY) - private TypeEntity type; + @Column(name = "type_id") + private String typeId; } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java index 7ce2ac61a..4bc477345 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java @@ -1,6 +1,9 @@ package com.iqser.red.service.persistence.management.v1.processor.service; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity; @@ -13,7 +16,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.Cl import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.storage.commons.service.StorageService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -25,9 +27,7 @@ import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; -import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert; @@ -57,7 +57,6 @@ public class DossierTemplateCloneService { DossierTemplateEntity clonedDossierTemplate = new DossierTemplateEntity(); - long start=System.currentTimeMillis(); dossierTemplateRepository.findById(dossierTemplateId).ifPresentOrElse((dossierTemplate) -> { OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS); @@ -72,7 +71,6 @@ public class DossierTemplateCloneService { clonedDossierTemplate.setValidTo(cloneDossierTemplateRequest.getValidTo() == null ? dossierTemplate.getValidTo() : cloneDossierTemplateRequest.getValidTo()); clonedDossierTemplate.setSoftDeleteTime(dossierTemplate.getSoftDeleteTime()); clonedDossierTemplate.setDownloadFileTypes(cloneDossierTemplateRequest.getDownloadFileTypes() == null || cloneDossierTemplateRequest.getDownloadFileTypes().isEmpty() ? dossierTemplate.getDownloadFileTypes() : cloneDossierTemplateRequest.getDownloadFileTypes()); - //set rules cloneRules(dossierTemplate.getId(), clonedDossierTemplate.getId()); //set legal basis @@ -83,21 +81,17 @@ public class DossierTemplateCloneService { dossierTemplateRepository.save(clonedDossierTemplate); //set dictionaries - long t1 = System.currentTimeMillis(); - clonedDossierTemplate.setDossierTypes(cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId())); - long t2 = System.currentTimeMillis(); - log.warn("Time to clone entries: {}", (t2 - t1)); + cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId()); //set dossier attributes cloneDossierAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId()); //set file attributes cloneFileAttributesGeneralConfig(dossierTemplate.getId(), clonedDossierTemplate.getId()); - - clonedDossierTemplate.setFileAttributeConfigs(cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId())); + cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId()); //set report templates - clonedDossierTemplate.setReportTemplates(cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId())); + cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId()); //set watermarks cloneWatermark(dossierTemplate.getId(), clonedDossierTemplate.getId()); @@ -108,14 +102,10 @@ public class DossierTemplateCloneService { //set dossier status cloneDossierStates(dossierTemplate.getId(), clonedDossierTemplate.getId()); - dossierTemplateRepository.save(clonedDossierTemplate); - }, () -> { throw new NotFoundException(String.format(dossierTemplatePersistenceService.DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE, dossierTemplateId)); }); - long end = System.currentTimeMillis(); - System.out.println("END TIME: "+(end-start)); - log.warn("End Time {}",end-start); + return clonedDossierTemplate; } @@ -132,23 +122,13 @@ public class DossierTemplateCloneService { } - private List cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) { + private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) { - List clonedTypes = new ArrayList<>(); var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false); for (TypeEntity t : types) { - TypeEntity te = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped()); - te.setDossierTemplateId(clonedDossierTemplateId); - clonedTypes.add(te); -// for (DictionaryEntryType det : DictionaryEntryType.values()) { -// var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null); -// Set entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet()); -// entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det); -// } - - entryPersistenceService.cloneEntries(dossierTemplateId,clonedDossierTemplateId); + var type = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped()); + entryPersistenceService.cloneEntries(t.getId(), type.getId()); } - return clonedTypes; } @@ -179,13 +159,13 @@ public class DossierTemplateCloneService { .filenameMappingColumnHeaderName(fileAttributesGeneralConfig.getFilenameMappingColumnHeaderName()) .build(); fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(clonedDossierTemplateId, fagc); - }catch (NotFoundException e){ + } catch (NotFoundException e) { log.debug("No file attribute config found for cloning {}", dossierTemplateId); } } - private List cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) { + private void cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) { List facList = new ArrayList<>(); var fileAttributesConfig = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId); @@ -203,14 +183,12 @@ public class DossierTemplateCloneService { facList.add(fac); }); fileAttributeConfigPersistenceService.setFileAttributesConfig(clonedDossierTemplateId, facList); - - return fileAttributeConfigPersistenceService.getFileAttributes(clonedDossierTemplateId); } - private List cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) { + private void cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) { - var reportTemplates = dossierTemplate.getReportTemplates(); + var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId()); for (ReportTemplateEntity rte : reportTemplates) { var storedReportTemplate = storageService.getObject(rte.getStorageId()); String storageId = clonedDossierTemplateId + "/" + rte.getFileName(); @@ -222,7 +200,6 @@ public class DossierTemplateCloneService { } reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport()); } - return reportTemplatePersistenceService.findByDossierTemplateId(clonedDossierTemplateId); } @@ -240,7 +217,7 @@ public class DossierTemplateCloneService { .orientation(watermark.getOrientation()) .build(); watermarkService.saveWatermark(clonedDossierTemplateId, we); - }catch (NotFoundException e){ + } catch (NotFoundException e) { log.debug("No watermark config found for cloning {}", dossierTemplateId); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java index 46f190892..e777d8916 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java @@ -8,6 +8,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository; +import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,6 +28,7 @@ public class EntryPersistenceService { private final TypeRepository typeRepository; private final FalsePositiveEntryRepository falsePositiveEntryRepository; private final FalseRecommendationEntryRepository falseRecommendationEntryRepository; + private final JDBCWriteUtils jdbcWriteUtils; @Transactional public void deleteEntries(String typeId, List values, long version, @@ -102,12 +104,12 @@ public class EntryPersistenceService { DictionaryEntryEntity entry = new DictionaryEntryEntity(); entry.setVersion(version); entry.setValue(e); - entry.setType(type); + entry.setTypeId(type.getId()); return entry; }).collect(Collectors.toList()); - entryRepository.saveAll(entryEntities); + jdbcWriteUtils.saveBatch(entryEntities); break; } case FALSE_POSITIVE: { @@ -118,12 +120,11 @@ public class EntryPersistenceService { DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity(); entry.setVersion(version); entry.setValue(e); - entry.setType(type); + entry.setTypeId(type.getId()); return entry; }).collect(Collectors.toList()); - - falsePositiveEntryRepository.saveAll(entryEntities); + jdbcWriteUtils.saveBatch(entryEntities); break; } case FALSE_RECOMMENDATION: { @@ -134,20 +135,19 @@ public class EntryPersistenceService { DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity(); entry.setVersion(version); entry.setValue(e); - entry.setType(type); + entry.setTypeId(type.getId()); return entry; }).collect(Collectors.toList()); - - falseRecommendationEntryRepository.saveAll(entryEntities); + jdbcWriteUtils.saveBatch(entryEntities); break; } } } - public void cloneEntries(String dossierTemplateId, String clonedDossierTemplateId) { - entryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId); -// falseRecommendationEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId); -// falsePositiveEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId); + public void cloneEntries(String originalTypeId, String newTypeId){ + entryRepository.cloneEntries(originalTypeId, newTypeId); + falseRecommendationEntryRepository.cloneEntries(originalTypeId, newTypeId); + falsePositiveEntryRepository.cloneEntries(originalTypeId, newTypeId); } } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java index 0812c3004..e5bf9160c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java @@ -12,24 +12,24 @@ import java.util.Set; public interface EntryRepository extends JpaRepository { @Modifying - @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id =:typeId and e.value in :values") + @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values") void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List values); @Modifying - @Query("update DictionaryEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false") + @Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false") void updateVersionWhereTypeId(long version, String typeId); @Modifying - @Query("update DictionaryEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId and e.deleted = false") + @Query("update DictionaryEntryEntity e set e.typeId = :newTypeId, e.version = e.version + 1 where e.typeId =:typeId and e.deleted = false") void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId); List findByTypeIdAndVersionGreaterThan(String typeId, long version); @Modifying @Transactional - @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId") + @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId") void deleteAllEntriesForTypeId(String typeId, long version); @@ -42,6 +42,6 @@ public interface EntryRepository extends JpaRepository { @Modifying - @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values") + @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values") void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List values); @Modifying - @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false") + @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false") void updateVersionWhereTypeId(long version, String typeId); @@ -25,11 +25,17 @@ public interface FalsePositiveEntryRepository extends JpaRepository undeleteEntries(String typeId, Set entries, long version); + + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true) + void cloneEntries(String originalTypeId, String newTypeId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java index 863442569..8e015876c 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java @@ -1,26 +1,23 @@ package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository; -import java.util.List; -import java.util.Set; - -import javax.transaction.Transactional; - +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity; +import javax.transaction.Transactional; +import java.util.List; +import java.util.Set; public interface FalseRecommendationEntryRepository extends JpaRepository { @Modifying - @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values") + @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values") void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List values); @Modifying - @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false") + @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false") void updateVersionWhereTypeId(long version, String typeId); @@ -28,11 +25,17 @@ public interface FalseRecommendationEntryRepository extends JpaRepository undeleteEntries(String typeId, Set entries, long version); + + @Modifying(flushAutomatically = true, clearAutomatically = true) + @Transactional + @Query(value = "insert into dictionary_false_recommendation_entry (value, version, deleted, type_id) " + + " select value, 1, false, :newTypeId from dictionary_false_recommendation_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true) + void cloneEntries(String originalTypeId, String newTypeId); } diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java index 38d631b2b..847656b24 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java @@ -39,7 +39,7 @@ public interface TypeRepository extends JpaRepository { @Query("select new com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e)) " + "from DossierTemplateEntity dt " + "left join TypeEntity t on t.dossierTemplateId = dt.id " + - "left join DictionaryEntryEntity e on e.type.id = t.id " + + "left join DictionaryEntryEntity e on e.typeId = t.id " + "where dt.id in :dossierTemplateIds and t.dossierId is null and (e.id is null or e.deleted = false) " + "group by dt.id, t.id, t.type, t.label ") List findDictionarySummaryList(Set dossierTemplateIds); diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java new file mode 100644 index 000000000..2b7d18a7d --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java @@ -0,0 +1,135 @@ +package com.iqser.red.service.persistence.management.v1.processor.utils.jdbc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +import javax.persistence.Column; +import javax.persistence.EntityManager; +import javax.persistence.Table; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.PreparedStatement; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.capitalize; + +@Service +@RequiredArgsConstructor +public class JDBCWriteUtils { + + private final JdbcTemplate jdbcTemplate; + + private final String SQL_TEMPLATE = "INSERT INTO %s (%s) values (%s)"; + + private final Map, EntityMetadata> entityMetadataMap = new HashMap<>(); + + private final EntityManager entityManager; + + public void saveBatch(final List entities) { + if (entities.isEmpty()) { + return; + } + + var metadata = getEntityMetadata(entities.iterator().next()); + + final int batchSize = 500; + + for (int j = 0; j < entities.size(); j += batchSize) { + + final List batchList = entities.subList(j, Math.min(j + batchSize, entities.size())); + + jdbcTemplate.batchUpdate(metadata.getSqlStatement(), + new BatchPreparedStatementSetter() { + @SneakyThrows + @Override + public void setValues(PreparedStatement ps, int i) { + T entity = batchList.get(i); + + int paramIndex = 1; + for (var mapping : metadata.getFieldMethodMap().entrySet()) { + ps.setObject(paramIndex++, mapping.getValue().invoke(entity)); + } + } + + @Override + public int getBatchSize() { + return batchList.size(); + } + }); + + } + + entityManager.clear(); + } + + private EntityMetadata getEntityMetadata(T entity) { + + var existingMetadata = entityMetadataMap.get(entity.getClass()); + if (existingMetadata != null) { + return existingMetadata; + } + + var tableName = getTableName(entity); + var args = getArgs(entity); + var sql = String.format(SQL_TEMPLATE, tableName, String.join(", ", args.keySet()), args.keySet().stream().map(a -> "?").collect(Collectors.joining(", "))); + + var metadata = new EntityMetadata(tableName, sql, args); + entityMetadataMap.put(entity.getClass(), metadata); + + return metadata; + } + + @SneakyThrows + private Map getArgs(T entity) { + var fields = entity.getClass().getDeclaredFields(); + Map entityMethodMap = new LinkedHashMap<>(); + for (var field : fields) { + var annotations = field.getDeclaredAnnotations(); + for (var annotation : annotations) { + if (annotation.annotationType().equals(Column.class)) { + var columnAnnotation = (Column) annotation; + var name = StringUtils.isEmpty(columnAnnotation.name()) ? toSnakeCase(field.getName()) : columnAnnotation.name(); + entityMethodMap.put(name, entity.getClass().getMethod(getMethodName(field))); + } + } + } + return entityMethodMap; + } + + private String getMethodName(Field field) { + var prefix = "get"; + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + prefix = "is"; + } + return prefix + capitalize(field.getName()); + } + + private String toSnakeCase(String name) { + String ret = name.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2"); + return ret.toLowerCase(); + } + + private String getTableName(T entity) { + var tableAnnot = entity.getClass().getDeclaredAnnotation(Table.class); + return tableAnnot.name(); + } + + @Data + @AllArgsConstructor + public static class EntityMetadata { + + private String tableName; + private String sqlStatement; + private Map fieldMethodMap; + } +} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java index 49cd8b485..2013a378d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java @@ -77,9 +77,7 @@ public class DictionaryController implements DictionaryResource { if (removeCurrent) { entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType); - long t1 = System.currentTimeMillis(); entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); - System.out.println("STUFF: {} "+(System.currentTimeMillis()-t1)); } else { entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml index 9fabdb7a9..9b5420755 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml @@ -13,7 +13,7 @@ spring: main: allow-circular-references: true # FIXME datasource: - url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction} + url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction}?cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true driverClassName: org.postgresql.Driver username: ${PSQL_USERNAME:redaction} password: ${PSQL_PASSWORD:redaction} diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml index 51706cdb7..5fb4b2cb2 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml @@ -71,3 +71,5 @@ databaseChangeLog: file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml - include: file: db/changelog/29-add-remove-digital-signatures-on-upload.changelog.yaml + - include: + file: db/changelog/sql/30-change-bigint-to-serial.sql diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/30-change-bigint-to-serial.sql b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/30-change-bigint-to-serial.sql new file mode 100644 index 000000000..3cdcbf4cb --- /dev/null +++ b/persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/30-change-bigint-to-serial.sql @@ -0,0 +1,7 @@ +ALTER TABLE dictionary_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY; +ALTER TABLE dictionary_false_recommendation_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY; +ALTER TABLE dictionary_false_positive_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY; + +SELECT setval(pg_get_serial_sequence('dictionary_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_entry)); +SELECT setval(pg_get_serial_sequence('dictionary_false_recommendation_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_recommendation_entry)); +SELECT setval(pg_get_serial_sequence('dictionary_false_positive_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_positive_entry)); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java index 88dd7db83..91702b310 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java @@ -8,12 +8,16 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType; +import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.List; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +@Slf4j public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest { @Autowired @@ -42,24 +46,38 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest var tenKEntries = generateEntries(10000); long t1 = System.currentTimeMillis(); - dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY); + dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY); long t2 = System.currentTimeMillis(); - System.out.println("Time T1: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries"); + log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size()); t1 = System.currentTimeMillis(); - dictionaryClient.addEntries(type.getTypeId(),tenKEntries , true, false, DictionaryEntryType.ENTRY); + dictionaryClient.addEntries(type.getTypeId(), tenKEntries, true, false, DictionaryEntryType.ENTRY); t2 = System.currentTimeMillis(); - System.out.println("Time T2: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries"); + log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size()); t1 = System.currentTimeMillis(); - dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY); + dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY); t2 = System.currentTimeMillis(); - System.out.println("Time T3: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries"); + log.info("Update Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size()); - dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest()); + dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_RECOMMENDATION); + dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_POSITIVE); + + t1 = System.currentTimeMillis(); + var cloned = dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest()); + t2 = System.currentTimeMillis(); + log.info("Clone Time: {}", (t2 - t1)); + + var types = dictionaryClient.getAllTypesForDossierTemplate(cloned.getId(), false); + + assertThat(types.size()).isEqualTo(1); + for (var clonedType : types) { + var found = entryRepository.findByTypeIdAndVersionGreaterThan(clonedType.getTypeId(), 0); + assertThat(found.size()).isEqualTo(5000); + } } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java index 3b66a1ac3..d0f4c6b11 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java @@ -100,7 +100,7 @@ public class FilePerformanceTest extends AbstractPersistenceServerServiceTest { List des = new ArrayList<>(); for (int j = 0; j < TYPE_ENTRY_COUNT; j++) { DictionaryEntryEntity de = new DictionaryEntryEntity(); - de.setType(te); + de.setTypeId(te.getId()); de.setDeleted(false); de.setVersion(1); de.setValue("Dictionary Entry" +i+"/"+ j); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackArchiverTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackArchiverTest.java index aed66fb73..0900841ec 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackArchiverTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackArchiverTest.java @@ -24,7 +24,7 @@ public class FileSystemBackArchiverTest { SplittableRandom sr = new SplittableRandom(); - var data = sr.doubles().limit(1024 * 1024 * 64).toArray(); + var data = sr.doubles().limit(1024 * 1024).toArray(); for (int i = 0; i < 10; i++) { log.info("At entry: {}, using {}MB of memory", i, (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024)); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml index 71b34d867..00c905c5a 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml +++ b/persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml @@ -77,3 +77,5 @@ management: metrics.enabled: false health.enabled: true endpoints.web.exposure.include: prometheus, health, metrics + +logging.level.root: info