RED-3942 - Improve clone performance

persistence-service-v1/persistence-service-processor-v1/pom.xml

@@ -100,6 +100,5 @@
             <artifactId>guava</artifactId>
             <scope>compile</scope>
         </dependency>
-
     </dependencies>
 </project>

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/BaseDictionaryEntry.java

@@ -4,13 +4,10 @@ public interface BaseDictionaryEntry {
 
     String getValue();
 
-
     long getVersion();
 
-
     boolean isDeleted();
 
-
+    String getTypeId();
-    TypeEntity getType();
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java

@@ -17,7 +17,7 @@ import javax.persistence.*;
 public class DictionaryEntryEntity implements BaseDictionaryEntry {
 
     @Id
-    @GeneratedValue
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
     private long entryId;
     @Column(length = 4000)
     private String value;
@@ -25,9 +25,8 @@ public class DictionaryEntryEntity implements BaseDictionaryEntry {
     private long version;
     @Column
     private boolean deleted;
-    @JsonIgnore
+    @Column(name = "type_id")
-    @ManyToOne(fetch = FetchType.LAZY)
+    private String typeId;
-    private TypeEntity type;
 
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalsePositiveEntryEntity.java

@@ -1,12 +1,6 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
-import javax.persistence.Column;
+import javax.persistence.*;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.GeneratedValue;
-import javax.persistence.Id;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
@@ -24,7 +18,7 @@ import lombok.NoArgsConstructor;
 public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry {
 
     @Id
-    @GeneratedValue
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
     private long entryId;
     @Column(length = 4000)
     private String value;
@@ -32,10 +26,8 @@ public class DictionaryFalsePositiveEntryEntity implements BaseDictionaryEntry {
     private long version;
     @Column
     private boolean deleted;
-
+    @Column( name = "type_id")
-    @JsonIgnore
+    private String typeId;
-    @ManyToOne(fetch = FetchType.LAZY)
-    private TypeEntity type;
 
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryFalseRecommendationEntryEntity.java

@@ -1,20 +1,12 @@
 package com.iqser.red.service.persistence.management.v1.processor.entity.configuration;
 
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.GeneratedValue;
-import javax.persistence.Id;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
+import javax.persistence.*;
+
 @Data
 @Builder
 @Entity
@@ -24,7 +16,7 @@ import lombok.NoArgsConstructor;
 public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryEntry {
 
     @Id
-    @GeneratedValue
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
     private long entryId;
     @Column(length = 4000)
     private String value;
@@ -32,10 +24,8 @@ public class DictionaryFalseRecommendationEntryEntity implements BaseDictionaryE
     private long version;
     @Column
     private boolean deleted;
-
+    @Column(name = "type_id")
-    @JsonIgnore
+    private String typeId;
-    @ManyToOne(fetch = FetchType.LAZY)
-    private TypeEntity type;
 
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.persistence.management.v1.processor.service;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
 import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
@@ -13,7 +16,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.Cl
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.storage.commons.service.StorageService;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -25,9 +27,7 @@ import java.time.OffsetDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
 
@@ -57,7 +57,6 @@ public class DossierTemplateCloneService {
 
         DossierTemplateEntity clonedDossierTemplate = new DossierTemplateEntity();
 
-        long start=System.currentTimeMillis();
         dossierTemplateRepository.findById(dossierTemplateId).ifPresentOrElse((dossierTemplate) -> {
 
             OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
@@ -72,7 +71,6 @@ public class DossierTemplateCloneService {
             clonedDossierTemplate.setValidTo(cloneDossierTemplateRequest.getValidTo() == null ? dossierTemplate.getValidTo() : cloneDossierTemplateRequest.getValidTo());
             clonedDossierTemplate.setSoftDeleteTime(dossierTemplate.getSoftDeleteTime());
             clonedDossierTemplate.setDownloadFileTypes(cloneDossierTemplateRequest.getDownloadFileTypes() == null || cloneDossierTemplateRequest.getDownloadFileTypes().isEmpty() ? dossierTemplate.getDownloadFileTypes() : cloneDossierTemplateRequest.getDownloadFileTypes());
-
             //set rules
             cloneRules(dossierTemplate.getId(), clonedDossierTemplate.getId());
             //set legal basis
@@ -83,21 +81,17 @@ public class DossierTemplateCloneService {
             dossierTemplateRepository.save(clonedDossierTemplate);
 
             //set dictionaries
-            long t1 = System.currentTimeMillis();
+            cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId());
-            clonedDossierTemplate.setDossierTypes(cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId()));
-            long t2 = System.currentTimeMillis();
-            log.warn("Time to clone entries: {}", (t2 - t1));
 
             //set dossier attributes
             cloneDossierAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
 
             //set file attributes
             cloneFileAttributesGeneralConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
-
+            cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
-            clonedDossierTemplate.setFileAttributeConfigs(cloneFileAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId()));
 
             //set report templates
-            clonedDossierTemplate.setReportTemplates(cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId()));
+            cloneReportTemplates(dossierTemplate, clonedDossierTemplate.getId());
 
             //set watermarks
             cloneWatermark(dossierTemplate.getId(), clonedDossierTemplate.getId());
@@ -108,14 +102,10 @@ public class DossierTemplateCloneService {
             //set dossier status
             cloneDossierStates(dossierTemplate.getId(), clonedDossierTemplate.getId());
 
-            dossierTemplateRepository.save(clonedDossierTemplate);
-
         }, () -> {
             throw new NotFoundException(String.format(dossierTemplatePersistenceService.DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE, dossierTemplateId));
         });
-        long end = System.currentTimeMillis();
+
-        System.out.println("END TIME: "+(end-start));
-        log.warn("End Time {}",end-start);
         return clonedDossierTemplate;
     }
 
@@ -132,23 +122,13 @@ public class DossierTemplateCloneService {
     }
 
 
-    private List<TypeEntity> cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
+    private void cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
 
-        List<TypeEntity> clonedTypes = new ArrayList<>();
         var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
         for (TypeEntity t : types) {
-            TypeEntity te = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped());
+            var type = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped());
-            te.setDossierTemplateId(clonedDossierTemplateId);
+            entryPersistenceService.cloneEntries(t.getId(), type.getId());
-            clonedTypes.add(te);
-//            for (DictionaryEntryType det : DictionaryEntryType.values()) {
-//                var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null);
-//                Set<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet());
-//                entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det);
-//            }
-
-            entryPersistenceService.cloneEntries(dossierTemplateId,clonedDossierTemplateId);
         }
-        return clonedTypes;
     }
 
 
@@ -179,13 +159,13 @@ public class DossierTemplateCloneService {
                     .filenameMappingColumnHeaderName(fileAttributesGeneralConfig.getFilenameMappingColumnHeaderName())
                     .build();
             fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(clonedDossierTemplateId, fagc);
-        }catch (NotFoundException e){
+        } catch (NotFoundException e) {
             log.debug("No file attribute config found for cloning {}", dossierTemplateId);
         }
     }
 
 
-    private List<FileAttributeConfigEntity> cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) {
+    private void cloneFileAttributesConfig(String dossierTemplateId, String clonedDossierTemplateId) {
 
         List<FileAttributeConfigEntity> facList = new ArrayList<>();
         var fileAttributesConfig = fileAttributeConfigPersistenceService.getFileAttributes(dossierTemplateId);
@@ -203,14 +183,12 @@ public class DossierTemplateCloneService {
             facList.add(fac);
         });
         fileAttributeConfigPersistenceService.setFileAttributesConfig(clonedDossierTemplateId, facList);
-
-        return fileAttributeConfigPersistenceService.getFileAttributes(clonedDossierTemplateId);
     }
 
 
-    private List<ReportTemplateEntity> cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) {
+    private void cloneReportTemplates(DossierTemplateEntity dossierTemplate, String clonedDossierTemplateId) {
 
-        var reportTemplates = dossierTemplate.getReportTemplates();
+        var reportTemplates = reportTemplatePersistenceService.findByDossierTemplateId(dossierTemplate.getId());
         for (ReportTemplateEntity rte : reportTemplates) {
             var storedReportTemplate = storageService.getObject(rte.getStorageId());
             String storageId = clonedDossierTemplateId + "/" + rte.getFileName();
@@ -222,7 +200,6 @@ public class DossierTemplateCloneService {
             }
             reportTemplatePersistenceService.insert(clonedDossierTemplateId, templateId, storageId, rte.getFileName(), rte.isActiveByDefault(), rte.isMultiFileReport());
         }
-        return reportTemplatePersistenceService.findByDossierTemplateId(clonedDossierTemplateId);
     }
 
 
@@ -240,7 +217,7 @@ public class DossierTemplateCloneService {
                     .orientation(watermark.getOrientation())
                     .build();
             watermarkService.saveWatermark(clonedDossierTemplateId, we);
-        }catch (NotFoundException e){
+        } catch (NotFoundException e) {
             log.debug("No watermark config found for cloning {}", dossierTemplateId);
         }
     }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java

@@ -8,6 +8,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
+import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -27,6 +28,7 @@ public class EntryPersistenceService {
     private final TypeRepository typeRepository;
     private final FalsePositiveEntryRepository falsePositiveEntryRepository;
     private final FalseRecommendationEntryRepository falseRecommendationEntryRepository;
+    private final JDBCWriteUtils jdbcWriteUtils;
 
     @Transactional
     public void deleteEntries(String typeId, List<String> values, long version,
@@ -102,12 +104,12 @@ public class EntryPersistenceService {
                     DictionaryEntryEntity entry = new DictionaryEntryEntity();
                     entry.setVersion(version);
                     entry.setValue(e);
-                    entry.setType(type);
+                    entry.setTypeId(type.getId());
                     return entry;
 
                 }).collect(Collectors.toList());
 
-                entryRepository.saveAll(entryEntities);
+                jdbcWriteUtils.saveBatch(entryEntities);
                 break;
             }
             case FALSE_POSITIVE: {
@@ -118,12 +120,11 @@ public class EntryPersistenceService {
                     DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity();
                     entry.setVersion(version);
                     entry.setValue(e);
-                    entry.setType(type);
+                    entry.setTypeId(type.getId());
                     return entry;
 
                 }).collect(Collectors.toList());
-
+                jdbcWriteUtils.saveBatch(entryEntities);
-                falsePositiveEntryRepository.saveAll(entryEntities);
                 break;
             }
             case FALSE_RECOMMENDATION: {
@@ -134,20 +135,19 @@ public class EntryPersistenceService {
                     DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity();
                     entry.setVersion(version);
                     entry.setValue(e);
-                    entry.setType(type);
+                    entry.setTypeId(type.getId());
                     return entry;
 
                 }).collect(Collectors.toList());
-
+                jdbcWriteUtils.saveBatch(entryEntities);
-                falseRecommendationEntryRepository.saveAll(entryEntities);
                 break;
             }
         }
     }
 
-    public void cloneEntries(String dossierTemplateId, String clonedDossierTemplateId) {
+    public void cloneEntries(String originalTypeId, String newTypeId){
-        entryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
+        entryRepository.cloneEntries(originalTypeId, newTypeId);
-//        falseRecommendationEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
+        falseRecommendationEntryRepository.cloneEntries(originalTypeId, newTypeId);
-//        falsePositiveEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
+        falsePositiveEntryRepository.cloneEntries(originalTypeId, newTypeId);
     }
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java

@@ -12,24 +12,24 @@ import java.util.Set;
 public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Long> {
 
     @Modifying
-    @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id =:typeId and e.value in :values")
+    @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
     void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
 
 
     @Modifying
-    @Query("update DictionaryEntryEntity e set e.version = :version where e.type.id =:typeId  and e.deleted = false")
+    @Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId  and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);
 
 
     @Modifying
-    @Query("update DictionaryEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId and e.deleted = false")
+    @Query("update DictionaryEntryEntity e set e.typeId = :newTypeId, e.version = e.version + 1 where e.typeId =:typeId and e.deleted = false")
     void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
 
     List<DictionaryEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
 
     @Modifying
     @Transactional
-    @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
+    @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
     void deleteAllEntriesForTypeId(String typeId, long version);
 
 
@@ -42,6 +42,6 @@ public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Lo
     @Modifying(flushAutomatically = true, clearAutomatically = true)
     @Transactional
     @Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " +
-            " select value, 1, false, :targetTypeId from dictionary_entry where type_id = :originTypeId and deleted = false", nativeQuery = true)
+            " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
-    void cloneEntries(String targetTypeId, String originTypeId);
+    void cloneEntries(String originalTypeId, String newTypeId);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java

@@ -12,12 +12,12 @@ import java.util.Set;
 public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
 
     @Modifying
-    @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
+    @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
     void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
 
 
     @Modifying
-    @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
+    @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);
 
 
@@ -25,11 +25,17 @@ public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFa
 
     @Modifying
     @Transactional
-    @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
+    @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
     void deleteAllEntriesForTypeId(String typeId, long version);
 
     @Modifying
     @Transactional
     @Query(value = "update dictionary_false_positive_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
     List<String> undeleteEntries(String typeId, Set<String> entries, long version);
+
+    @Modifying(flushAutomatically = true, clearAutomatically = true)
+    @Transactional
+    @Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " +
+            " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
+    void cloneEntries(String originalTypeId, String newTypeId);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java

@@ -1,26 +1,23 @@
 package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
 
-import java.util.List;
+import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
-import java.util.Set;
-
-import javax.transaction.Transactional;
-
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
+import javax.transaction.Transactional;
-import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
+import java.util.List;
+import java.util.Set;
 
 public interface FalseRecommendationEntryRepository extends JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
 
     @Modifying
-    @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
+    @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
     void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
 
 
     @Modifying
-    @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
+    @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);
 
 
@@ -28,11 +25,17 @@ public interface FalseRecommendationEntryRepository extends JpaRepository<Dictio
 
     @Modifying
     @Transactional
-    @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
+    @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
     void deleteAllEntriesForTypeId(String typeId, long version);
 
     @Modifying
     @Transactional
     @Query(value = "update dictionary_false_recommendation_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
     List<String> undeleteEntries(String typeId, Set<String> entries, long version);
+
+    @Modifying(flushAutomatically = true, clearAutomatically = true)
+    @Transactional
+    @Query(value = "insert into dictionary_false_recommendation_entry (value, version, deleted, type_id) " +
+            " select value, 1, false, :newTypeId from dictionary_false_recommendation_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)
+    void cloneEntries(String originalTypeId, String newTypeId);
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/TypeRepository.java

@@ -39,7 +39,7 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
     @Query("select new com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse(dt.id, t.id, t.type, t.label, count(e)) " +
             "from DossierTemplateEntity dt " +
             "left join TypeEntity t on t.dossierTemplateId = dt.id " +
-            "left join DictionaryEntryEntity e on e.type.id = t.id " +
+            "left join DictionaryEntryEntity e on e.typeId = t.id " +
             "where dt.id in :dossierTemplateIds and t.dossierId is null and (e.id is null or e.deleted = false) " +
             "group by dt.id, t.id, t.type, t.label ")
     List<DictionarySummaryResponse> findDictionarySummaryList(Set<String> dossierTemplateIds);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/jdbc/JDBCWriteUtils.java

@@ -0,0 +1,135 @@
+package com.iqser.red.service.persistence.management.v1.processor.utils.jdbc;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+
+import javax.persistence.Column;
+import javax.persistence.EntityManager;
+import javax.persistence.Table;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.sql.PreparedStatement;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.commons.lang3.StringUtils.capitalize;
+
+@Service
+@RequiredArgsConstructor
+public class JDBCWriteUtils {
+
+    private final JdbcTemplate jdbcTemplate;
+
+    private final String SQL_TEMPLATE = "INSERT INTO %s (%s) values (%s)";
+
+    private final Map<Class<?>, EntityMetadata> entityMetadataMap = new HashMap<>();
+
+    private final EntityManager entityManager;
+
+    public <T> void saveBatch(final List<T> entities) {
+        if (entities.isEmpty()) {
+            return;
+        }
+
+        var metadata = getEntityMetadata(entities.iterator().next());
+
+        final int batchSize = 500;
+
+        for (int j = 0; j < entities.size(); j += batchSize) {
+
+            final List<T> batchList = entities.subList(j, Math.min(j + batchSize, entities.size()));
+
+            jdbcTemplate.batchUpdate(metadata.getSqlStatement(),
+                    new BatchPreparedStatementSetter() {
+                        @SneakyThrows
+                        @Override
+                        public void setValues(PreparedStatement ps, int i) {
+                            T entity = batchList.get(i);
+
+                            int paramIndex = 1;
+                            for (var mapping : metadata.getFieldMethodMap().entrySet()) {
+                                ps.setObject(paramIndex++, mapping.getValue().invoke(entity));
+                            }
+                        }
+
+                        @Override
+                        public int getBatchSize() {
+                            return batchList.size();
+                        }
+                    });
+
+        }
+
+        entityManager.clear();
+    }
+
+    private <T> EntityMetadata getEntityMetadata(T entity) {
+
+        var existingMetadata = entityMetadataMap.get(entity.getClass());
+        if (existingMetadata != null) {
+            return existingMetadata;
+        }
+
+        var tableName = getTableName(entity);
+        var args = getArgs(entity);
+        var sql = String.format(SQL_TEMPLATE, tableName, String.join(", ", args.keySet()), args.keySet().stream().map(a -> "?").collect(Collectors.joining(", ")));
+
+        var metadata = new EntityMetadata(tableName, sql, args);
+        entityMetadataMap.put(entity.getClass(), metadata);
+
+        return metadata;
+    }
+
+    @SneakyThrows
+    private <T> Map<String, Method> getArgs(T entity) {
+        var fields = entity.getClass().getDeclaredFields();
+        Map<String, Method> entityMethodMap = new LinkedHashMap<>();
+        for (var field : fields) {
+            var annotations = field.getDeclaredAnnotations();
+            for (var annotation : annotations) {
+                if (annotation.annotationType().equals(Column.class)) {
+                    var columnAnnotation = (Column) annotation;
+                    var name = StringUtils.isEmpty(columnAnnotation.name()) ? toSnakeCase(field.getName()) : columnAnnotation.name();
+                    entityMethodMap.put(name, entity.getClass().getMethod(getMethodName(field)));
+                }
+            }
+        }
+        return entityMethodMap;
+    }
+
+    private String getMethodName(Field field) {
+        var prefix = "get";
+        if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) {
+            prefix = "is";
+        }
+        return prefix + capitalize(field.getName());
+    }
+
+    private String toSnakeCase(String name) {
+        String ret = name.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2");
+        return ret.toLowerCase();
+    }
+
+    private <T> String getTableName(T entity) {
+        var tableAnnot = entity.getClass().getDeclaredAnnotation(Table.class);
+        return tableAnnot.name();
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class EntityMetadata {
+
+        private String tableName;
+        private String sqlStatement;
+        private Map<String, Method> fieldMethodMap;
+    }
+}

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java

@@ -77,9 +77,7 @@ public class DictionaryController implements DictionaryResource {
 
         if (removeCurrent) {
             entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
-            long t1 = System.currentTimeMillis();
             entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
-            System.out.println("STUFF: {} "+(System.currentTimeMillis()-t1));
         } else {
             entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
         }

persistence-service-v1/persistence-service-server-v1/src/main/resources/application.yml

@@ -13,7 +13,7 @@ spring:
   main:
     allow-circular-references: true # FIXME
   datasource:
-    url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction}
+    url: jdbc:postgresql://${PSQL_HOST:localhost}:${PSQL_PORT:5432}/${PSQL_DATABASE:redaction}?cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true
     driverClassName: org.postgresql.Driver
     username: ${PSQL_USERNAME:redaction}
     password: ${PSQL_PASSWORD:redaction}

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -71,3 +71,5 @@ databaseChangeLog:
       file: db/changelog/28-add-update-dictionary-to-manual-resize-redactions.yaml
   - include:
       file: db/changelog/29-add-remove-digital-signatures-on-upload.changelog.yaml
+  - include:
+      file: db/changelog/sql/30-change-bigint-to-serial.sql

persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/sql/30-change-bigint-to-serial.sql

@@ -0,0 +1,7 @@
+ALTER TABLE dictionary_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
+ALTER TABLE dictionary_false_recommendation_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
+ALTER TABLE dictionary_false_positive_entry ALTER entry_id ADD GENERATED ALWAYS AS IDENTITY;
+
+SELECT setval(pg_get_serial_sequence('dictionary_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_entry));
+SELECT setval(pg_get_serial_sequence('dictionary_false_recommendation_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_recommendation_entry));
+SELECT setval(pg_get_serial_sequence('dictionary_false_positive_entry', 'entry_id'), (select coalesce(max(entry_id)+1,1) from dictionary_false_positive_entry));

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java

@@ -8,12 +8,16 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers
 import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
+import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+@Slf4j
 public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest {
 
     @Autowired
@@ -42,24 +46,38 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
         var tenKEntries = generateEntries(10000);
 
         long t1 = System.currentTimeMillis();
-        dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
+        dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY);
         long t2 = System.currentTimeMillis();
-        System.out.println("Time T1: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
+        log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
 
 
         t1 = System.currentTimeMillis();
-        dictionaryClient.addEntries(type.getTypeId(),tenKEntries , true, false, DictionaryEntryType.ENTRY);
+        dictionaryClient.addEntries(type.getTypeId(), tenKEntries, true, false, DictionaryEntryType.ENTRY);
         t2 = System.currentTimeMillis();
-        System.out.println("Time T2: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
+        log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
 
 
         t1 = System.currentTimeMillis();
-        dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
+        dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.ENTRY);
         t2 = System.currentTimeMillis();
-        System.out.println("Time T3: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
+        log.info("Update Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
 
 
-        dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
+        dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_RECOMMENDATION);
+        dictionaryClient.addEntries(type.getTypeId(), fiveKEntries, true, false, DictionaryEntryType.FALSE_POSITIVE);
+
+        t1 = System.currentTimeMillis();
+        var cloned = dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
+        t2 = System.currentTimeMillis();
+        log.info("Clone Time: {}", (t2 - t1));
+
+        var types = dictionaryClient.getAllTypesForDossierTemplate(cloned.getId(), false);
+
+        assertThat(types.size()).isEqualTo(1);
+        for (var clonedType : types) {
+            var found = entryRepository.findByTypeIdAndVersionGreaterThan(clonedType.getTypeId(), 0);
+            assertThat(found.size()).isEqualTo(5000);
+        }
 
     }
 

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java

@@ -100,7 +100,7 @@ public class FilePerformanceTest extends AbstractPersistenceServerServiceTest {
             List<DictionaryEntryEntity> des = new ArrayList<>();
             for (int j = 0; j < TYPE_ENTRY_COUNT; j++) {
                 DictionaryEntryEntity de = new DictionaryEntryEntity();
-                de.setType(te);
+                de.setTypeId(te.getId());
                 de.setDeleted(false);
                 de.setVersion(1);
                 de.setValue("Dictionary Entry" +i+"/"+ j);

persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/utils/FileSystemBackArchiverTest.java

@@ -24,7 +24,7 @@ public class FileSystemBackArchiverTest {
 
             SplittableRandom sr = new SplittableRandom();
 
-            var data = sr.doubles().limit(1024 * 1024 * 64).toArray();
+            var data = sr.doubles().limit(1024 * 1024).toArray();
             for (int i = 0; i < 10; i++) {
                 log.info("At entry: {}, using {}MB of memory", i, (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024));
 

persistence-service-v1/persistence-service-server-v1/src/test/resources/application.yml

@@ -77,3 +77,5 @@ management:
     metrics.enabled: false
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health, metrics
+
+logging.level.root: info