Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6777: Reimplemented deletion of dictionary entries as a batch process to avoid a limitation in the Postgres JDBC driver

Browse Source 
(change backported from master)
This commit is contained in:
Viktor Seifert 2023-05-22 15:40:14 +02:00
parent 1cb2cc1700
commit 71a11fc24a
12 changed files with 86 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java
View File

@ -35,18 +35,12 @@ public class EntryPersistenceService {
@Transactional
public void deleteEntries(String typeId, List<String> values, long version, DictionaryEntryType dictionaryEntryType) {
public void deleteEntries(String typeId, Set<String> values, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case ENTRY -> entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
}
}
@ -55,45 +49,29 @@ public class EntryPersistenceService {
public void setVersion(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
case ENTRY -> entryRepository.updateVersionWhereTypeId(version, typeId);
case FALSE_POSITIVE -> falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
}
}
public List<? extends BaseDictionaryEntry> getEntries(String typeId, DictionaryEntryType dictionaryEntryType, Long fromVersion) {
switch (dictionaryEntryType) {
case ENTRY:
return entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_POSITIVE:
return falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_RECOMMENDATION:
return falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
}
return null;
return switch (dictionaryEntryType) {
case ENTRY -> entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_POSITIVE -> falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
};
}
public void deleteAllEntriesForTypeId(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case ENTRY -> entryRepository.deleteAllEntriesForTypeId(typeId, version);
case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java
View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<DictionaryEntryEntity, Long> {
@Modifying
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
@ -35,6 +30,7 @@ public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<Di
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryCustom.java
View File

@ -7,4 +7,7 @@ public interface EntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryImpl.java
View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
public class EntryRepositoryImpl implements EntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

6
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepository.java
View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface FalsePositiveEntryRepository extends FalsePositiveEntryRepositoryCustom, JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
@ -30,6 +25,7 @@ public interface FalsePositiveEntryRepository extends FalsePositiveEntryReposito
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryCustom.java
View File

@ -7,4 +7,7 @@ public interface FalsePositiveEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryImpl.java
View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
class FalsePositiveEntryRepositoryImpl implements FalsePositiveEntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_false_positive_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_positive_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepository.java
View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface FalseRecommendationEntryRepository extends FalseRecommendationEntryRepositoryCustom, JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);

3
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryCustom.java
View File

@ -7,4 +7,7 @@ public interface FalseRecommendationEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryImpl.java
View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
class FalseRecommendationEntryRepositoryImpl implements FalseRecommendationEntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_false_recommendation_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_recommendation_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

31
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/QueryExecutor.java
View File

@ -26,7 +26,7 @@ class QueryExecutor {
private static final String UPDATE_ENTRIES_QUERY = """
update ::tableName::
set deleted = false, version = :version
set deleted = ::deleted::, version = :version
where type_id = :typeId and value in (:entries)""";
// Currently (2023-04-13) there is a limitation in the Postgres JDBC driver, that limits the number of elements in a "IN" clause
@ -40,6 +40,12 @@ class QueryExecutor {
@Transactional
public LinkedList<String> runUndeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
return runUpdateQueryInBatches(typeId, entries, version, tableName, false, true);
}
private LinkedList<String> runUpdateQueryInBatches(String typeId, Set<String> entries, long version, String tableName, boolean deleted, boolean collectChangedValues) {
var results = new LinkedList<String>();
var entryList = new ArrayList<>(entries);
@ -53,10 +59,12 @@ class QueryExecutor {
var values = entryList.subList(fromIndex, toIndex);
var entryValues = executeFetchValuesQuery(typeId, tableName, values);
results.addAll(entryValues);
if (collectChangedValues) {
var entryValues = executeFetchValuesQuery(typeId, tableName, values);
results.addAll(entryValues);
}
executeUpdateQuery(typeId, version, tableName, values);
executeUpdateQuery(typeId, version, tableName, values, deleted);
fromIndex += ELEMENT_CHUNK_SIZE;
toIndex += ELEMENT_CHUNK_SIZE;
@ -66,9 +74,9 @@ class QueryExecutor {
}
private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values) {
private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values, boolean deleted) {
String updateSql = getUpdateEntriesQuery(tableName);
String updateSql = getUpdateEntriesQuery(tableName, deleted);
Query updateEntriesQuery = entityManager.createNativeQuery(updateSql);
updateEntriesQuery.setParameter("typeId", typeId);
@ -101,9 +109,16 @@ class QueryExecutor {
}
private String getUpdateEntriesQuery(String tableName) {
private String getUpdateEntriesQuery(String tableName, boolean deleted) {
return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName);
return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName).replace("::deleted::", Boolean.toString(deleted));
}
@Transactional
public void runDeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
runUpdateQueryInBatches(typeId, entries, version, tableName, true, false);
}
}

11
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/DictionaryService.java
View File

@ -1,9 +1,9 @@
package com.iqser.red.service.peristence.v1.server.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -11,6 +11,8 @@ import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.transaction.Transactional;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@ -41,6 +43,7 @@ public class DictionaryService {
private final StopwordService stopwordService;
@Transactional
public Type addType(Type typeRequest) {
if (typeRequest.getDossierTemplateId() == null) {
@ -182,13 +185,13 @@ public class DictionaryService {
var currentVersion = getCurrentVersion(typeResult);
if (typeResult.isCaseInsensitive()) {
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).collect(toList());
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).toList();
entryPersistenceService.deleteEntries(typeId,
existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toList()),
existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toSet()),
currentVersion + 1,
dictionaryEntryType);
} else {
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.deleteEntries(typeId, new HashSet<>(entries), currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);