RED-6777: Reimplemented deletion of dictionary entries as a batch process to avoid a limitation in the Postgres JDBC driver

(change backported from master)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java

@@ -35,18 +35,12 @@ public class EntryPersistenceService {
 
 
     @Transactional
-    public void deleteEntries(String typeId, List<String> values, long version, DictionaryEntryType dictionaryEntryType) {
+    public void deleteEntries(String typeId, Set<String> values, long version, DictionaryEntryType dictionaryEntryType) {
 
         switch (dictionaryEntryType) {
-            case ENTRY:
+            case ENTRY -> entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
-                entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
+            case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
-                break;
+            case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
-            case FALSE_POSITIVE:
-                falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
-                break;
-            case FALSE_RECOMMENDATION:
-                falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
-                break;
         }
     }
 
@@ -55,45 +49,29 @@ public class EntryPersistenceService {
     public void setVersion(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
 
         switch (dictionaryEntryType) {
-            case ENTRY:
+            case ENTRY -> entryRepository.updateVersionWhereTypeId(version, typeId);
-                entryRepository.updateVersionWhereTypeId(version, typeId);
+            case FALSE_POSITIVE -> falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
-                break;
+            case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
-            case FALSE_POSITIVE:
-                falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
-                break;
-            case FALSE_RECOMMENDATION:
-                falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
-                break;
         }
     }
 
 
     public List<? extends BaseDictionaryEntry> getEntries(String typeId, DictionaryEntryType dictionaryEntryType, Long fromVersion) {
 
-        switch (dictionaryEntryType) {
+        return switch (dictionaryEntryType) {
-            case ENTRY:
+            case ENTRY -> entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
-                return entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
+            case FALSE_POSITIVE -> falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
-            case FALSE_POSITIVE:
+            case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
-                return falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
+        };
-            case FALSE_RECOMMENDATION:
-                return falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
-        }
-        return null;
     }
 
 
     public void deleteAllEntriesForTypeId(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
 
         switch (dictionaryEntryType) {
-            case ENTRY:
+            case ENTRY -> entryRepository.deleteAllEntriesForTypeId(typeId, version);
-                entryRepository.deleteAllEntriesForTypeId(typeId, version);
+            case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
-                break;
+            case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
-            case FALSE_POSITIVE:
-                falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
-                break;
-            case FALSE_RECOMMENDATION:
-                falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
-                break;
         }
     }
 

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java

@@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
 
 public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<DictionaryEntryEntity, Long> {
 
-    @Modifying
-    @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
-    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
-
-
     @Modifying
     @Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId  and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);
@@ -35,6 +30,7 @@ public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<Di
     @Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
     void deleteAllEntriesForTypeId(String typeId, long version);
 
+
     @Modifying(flushAutomatically = true, clearAutomatically = true)
     @Transactional
     @Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryCustom.java

@@ -7,4 +7,7 @@ public interface EntryRepositoryCustom {
 
     List<String> undeleteEntries(String typeId, Set<String> entries, long version);
 
+
+    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryImpl.java

@@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
 @Repository
 public class EntryRepositoryImpl implements EntryRepositoryCustom {
 
+    private static final String TABLE_NAME = "dictionary_entry";
+
     QueryExecutor queryExecutor;
 
 
     @Override
     public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
 
-        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_entry");
+        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
+    }
+
+
+    @Override
+    public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
+
+        queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepository.java

@@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
 
 public interface FalsePositiveEntryRepository extends FalsePositiveEntryRepositoryCustom, JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
 
-    @Modifying
-    @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
-    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
-
-
     @Modifying
     @Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);
@@ -30,6 +25,7 @@ public interface FalsePositiveEntryRepository extends FalsePositiveEntryReposito
     @Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
     void deleteAllEntriesForTypeId(String typeId, long version);
 
+
     @Modifying(flushAutomatically = true, clearAutomatically = true)
     @Transactional
     @Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryCustom.java

@@ -7,4 +7,7 @@ public interface FalsePositiveEntryRepositoryCustom {
 
     List<String> undeleteEntries(String typeId, Set<String> entries, long version);
 
+
+    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryImpl.java

@@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
 @Repository
 class FalsePositiveEntryRepositoryImpl implements FalsePositiveEntryRepositoryCustom {
 
+    private static final String TABLE_NAME = "dictionary_false_positive_entry";
+
     QueryExecutor queryExecutor;
 
 
     @Override
     public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
 
-        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_positive_entry");
+        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
+    }
+
+
+    @Override
+    public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
+
+        queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepository.java

@@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
 
 public interface FalseRecommendationEntryRepository extends FalseRecommendationEntryRepositoryCustom, JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
 
-    @Modifying
-    @Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
-    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
-
-
     @Modifying
     @Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
     void updateVersionWhereTypeId(long version, String typeId);

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryCustom.java

@@ -7,4 +7,7 @@ public interface FalseRecommendationEntryRepositoryCustom {
 
     List<String> undeleteEntries(String typeId, Set<String> entries, long version);
 
+
+    void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
+
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryImpl.java

@@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
 @Repository
 class FalseRecommendationEntryRepositoryImpl implements FalseRecommendationEntryRepositoryCustom {
 
+    private static final String TABLE_NAME = "dictionary_false_recommendation_entry";
+
     QueryExecutor queryExecutor;
 
 
     @Override
     public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
 
-        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_recommendation_entry");
+        return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
+    }
+
+
+    @Override
+    public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
+
+        queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
     }
 
 }

persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/QueryExecutor.java

@@ -26,7 +26,7 @@ class QueryExecutor {
 
     private static final String UPDATE_ENTRIES_QUERY = """
             update ::tableName::
-                set deleted = false, version = :version
+                set deleted = ::deleted::, version = :version
                 where type_id = :typeId and value in (:entries)""";
 
     // Currently (2023-04-13) there is a limitation in the Postgres JDBC driver, that limits the number of elements in a "IN" clause
@@ -40,6 +40,12 @@ class QueryExecutor {
     @Transactional
     public LinkedList<String> runUndeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
 
+        return runUpdateQueryInBatches(typeId, entries, version, tableName, false, true);
+    }
+
+
+    private LinkedList<String> runUpdateQueryInBatches(String typeId, Set<String> entries, long version, String tableName, boolean deleted, boolean collectChangedValues) {
+
         var results = new LinkedList<String>();
 
         var entryList = new ArrayList<>(entries);
@@ -53,10 +59,12 @@ class QueryExecutor {
 
             var values = entryList.subList(fromIndex, toIndex);
 
-            var entryValues = executeFetchValuesQuery(typeId, tableName, values);
+            if (collectChangedValues) {
-            results.addAll(entryValues);
+                var entryValues = executeFetchValuesQuery(typeId, tableName, values);
+                results.addAll(entryValues);
+            }
 
-            executeUpdateQuery(typeId, version, tableName, values);
+            executeUpdateQuery(typeId, version, tableName, values, deleted);
 
             fromIndex += ELEMENT_CHUNK_SIZE;
             toIndex += ELEMENT_CHUNK_SIZE;
@@ -66,9 +74,9 @@ class QueryExecutor {
     }
 
 
-    private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values) {
+    private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values, boolean deleted) {
 
-        String updateSql = getUpdateEntriesQuery(tableName);
+        String updateSql = getUpdateEntriesQuery(tableName, deleted);
 
         Query updateEntriesQuery = entityManager.createNativeQuery(updateSql);
         updateEntriesQuery.setParameter("typeId", typeId);
@@ -101,9 +109,16 @@ class QueryExecutor {
     }
 
 
-    private String getUpdateEntriesQuery(String tableName) {
+    private String getUpdateEntriesQuery(String tableName, boolean deleted) {
 
-        return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName);
+        return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName).replace("::deleted::", Boolean.toString(deleted));
+    }
+
+
+    @Transactional
+    public void runDeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
+
+        runUpdateQueryInBatches(typeId, entries, version, tableName, true, false);
     }
 
 }

persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java

@@ -1,9 +1,9 @@
 package com.iqser.red.service.peristence.v1.server.service;
 
 import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
-import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -11,6 +11,8 @@ import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.transaction.Transactional;
+
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
@@ -41,6 +43,7 @@ public class DictionaryService {
     private final StopwordService stopwordService;
 
 
+    @Transactional
     public Type addType(Type typeRequest) {
 
         if (typeRequest.getDossierTemplateId() == null) {
@@ -182,13 +185,13 @@ public class DictionaryService {
         var currentVersion = getCurrentVersion(typeResult);
 
         if (typeResult.isCaseInsensitive()) {
-            List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).collect(toList());
+            List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).toList();
             entryPersistenceService.deleteEntries(typeId,
-                    existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toList()),
+                    existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toSet()),
                     currentVersion + 1,
                     dictionaryEntryType);
         } else {
-            entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
+            entryPersistenceService.deleteEntries(typeId, new HashSet<>(entries), currentVersion + 1, dictionaryEntryType);
         }
 
         dictionaryPersistenceService.incrementVersion(typeId);