Pull request #696: RED-6777

Merge in RED/persistence-service from RED-6777 to master

* commit 'ca6f3f54f201398452cfbfd044e3b014d499d4a9':
  RED-6777: Reordered parameters for consistency
  RED-6777: Reimplemented deletion of dictionary entries as a batch process to avoid a limitation in the Postgres JDBC driver
  RED-6777: Added a test for a dictionary update and delete with a large number of entries
This commit is contained in:
Viktor Seifert 2023-05-22 16:30:53 +02:00
commit 1c02f96c5a
13 changed files with 147 additions and 74 deletions

View File

@ -1,8 +1,8 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@ -273,13 +273,13 @@ public class DictionaryManagementService {
var currentVersion = getCurrentVersion(typeResult);
if (typeResult.isCaseInsensitive()) {
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).collect(toList());
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null).stream().map(BaseDictionaryEntry::getValue).toList();
entryPersistenceService.deleteEntries(typeId,
existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toList()),
existing.stream().filter(e -> entries.stream().anyMatch(e::equalsIgnoreCase)).collect(toSet()),
currentVersion + 1,
dictionaryEntryType);
} else {
entryPersistenceService.deleteEntries(typeId, entries, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.deleteEntries(typeId, new HashSet<>(entries), currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);

View File

@ -35,18 +35,12 @@ public class EntryPersistenceService {
@Transactional
public void deleteEntries(String typeId, List<String> values, long version, DictionaryEntryType dictionaryEntryType) {
public void deleteEntries(String typeId, Set<String> values, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, version, values);
break;
case ENTRY -> entryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllByTypeIdAndVersionAndValueIn(typeId, values, version);
}
}
@ -55,45 +49,29 @@ public class EntryPersistenceService {
public void setVersion(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
case ENTRY -> entryRepository.updateVersionWhereTypeId(version, typeId);
case FALSE_POSITIVE -> falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
}
}
public List<? extends BaseDictionaryEntry> getEntries(String typeId, DictionaryEntryType dictionaryEntryType, Long fromVersion) {
switch (dictionaryEntryType) {
case ENTRY:
return entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_POSITIVE:
return falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_RECOMMENDATION:
return falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
}
return null;
return switch (dictionaryEntryType) {
case ENTRY -> entryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_POSITIVE -> falsePositiveEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.findByTypeIdAndVersionGreaterThan(typeId, fromVersion != null ? fromVersion : -1);
};
}
public void deleteAllEntriesForTypeId(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case ENTRY -> entryRepository.deleteAllEntriesForTypeId(typeId, version);
case FALSE_POSITIVE -> falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
case FALSE_RECOMMENDATION -> falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
}
}

View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<DictionaryEntryEntity, Long> {
@Modifying
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
@ -35,6 +30,7 @@ public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<Di
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

View File

@ -7,4 +7,7 @@ public interface EntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
public class EntryRepositoryImpl implements EntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface FalsePositiveEntryRepository extends FalsePositiveEntryRepositoryCustom, JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
@ -30,6 +25,7 @@ public interface FalsePositiveEntryRepository extends FalsePositiveEntryReposito
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

View File

@ -7,4 +7,7 @@ public interface FalsePositiveEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
class FalsePositiveEntryRepositoryImpl implements FalsePositiveEntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_false_positive_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_positive_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

View File

@ -12,11 +12,6 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface FalseRecommendationEntryRepository extends FalseRecommendationEntryRepositoryCustom, JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.typeId = :typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);

View File

@ -7,4 +7,7 @@ public interface FalseRecommendationEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version);
}

View File

@ -14,13 +14,22 @@ import lombok.experimental.FieldDefaults;
@Repository
class FalseRecommendationEntryRepositoryImpl implements FalseRecommendationEntryRepositoryCustom {
private static final String TABLE_NAME = "dictionary_false_recommendation_entry";
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_recommendation_entry");
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
@Override
public void deleteAllByTypeIdAndVersionAndValueIn(String typeId, Set<String> entries, long version) {
queryExecutor.runDeleteQueryInBatches(typeId, entries, version, TABLE_NAME);
}
}

View File

@ -26,7 +26,7 @@ class QueryExecutor {
private static final String UPDATE_ENTRIES_QUERY = """
update ::tableName::
set deleted = false, version = :version
set deleted = ::deleted::, version = :version
where type_id = :typeId and value in (:entries)""";
// Currently (2023-04-13) there is a limitation in the Postgres JDBC driver, that limits the number of elements in a "IN" clause
@ -40,6 +40,12 @@ class QueryExecutor {
@Transactional
public LinkedList<String> runUndeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
return runUpdateQueryInBatches(typeId, entries, version, tableName, false, true);
}
private LinkedList<String> runUpdateQueryInBatches(String typeId, Set<String> entries, long version, String tableName, boolean deleted, boolean collectChangedValues) {
var results = new LinkedList<String>();
var entryList = new ArrayList<>(entries);
@ -53,10 +59,12 @@ class QueryExecutor {
var values = entryList.subList(fromIndex, toIndex);
var entryValues = executeFetchValuesQuery(typeId, tableName, values);
results.addAll(entryValues);
if (collectChangedValues) {
var entryValues = executeFetchValuesQuery(typeId, tableName, values);
results.addAll(entryValues);
}
executeUpdateQuery(typeId, version, tableName, values);
executeUpdateQuery(typeId, version, tableName, values, deleted);
fromIndex += ELEMENT_CHUNK_SIZE;
toIndex += ELEMENT_CHUNK_SIZE;
@ -66,9 +74,9 @@ class QueryExecutor {
}
private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values) {
private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values, boolean deleted) {
String updateSql = getUpdateEntriesQuery(tableName);
String updateSql = getUpdateEntriesQuery(tableName, deleted);
Query updateEntriesQuery = entityManager.createNativeQuery(updateSql);
updateEntriesQuery.setParameter("typeId", typeId);
@ -101,9 +109,16 @@ class QueryExecutor {
}
private String getUpdateEntriesQuery(String tableName) {
private String getUpdateEntriesQuery(String tableName, boolean deleted) {
return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName);
return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName).replace("::deleted::", Boolean.toString(deleted));
}
@Transactional
public void runDeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
runUpdateQueryInBatches(typeId, entries, version, tableName, true, false);
}
}

View File

@ -3,8 +3,9 @@ package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Test;
import org.springframework.beans.BeanUtils;
@ -19,7 +20,6 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.UpdateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import feign.FeignException;
@ -100,6 +100,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
}
}
@Test
public void testAddEntriesWithStopWord2() {
@ -285,7 +286,6 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var type = CreateTypeValue.builder()
.type("dossier_redaction")
.label("Dossier Redactions")
@ -297,7 +297,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.dossierTemplateId(dossierTemplate.getId())
.build();
var createdType = dictionaryClient.addType(type,null);
var createdType = dictionaryClient.addType(type, null);
var word1 = "Luke Skywalker";
var word2 = "Anakin Skywalker";
@ -314,11 +314,12 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
var actualEntries = dictionary.getEntries();
assertThat(actualEntries.size()).isEqualTo(3);
assertThat(actualEntries).usingComparator(new ListContentWithoutOrderAndWithoutDuplicatesComparator<>()).isEqualTo(entries);
dictionaryClient.deleteType(createdType.getType(), createdType.getDossierTemplateId(),null);
assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(),null,false).getTypes().size()).isEqualTo(0);
dictionaryClient.deleteType(createdType.getType(), createdType.getDossierTemplateId(), null);
assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(), null, false).getTypes().size()).isEqualTo(0);
dictionaryClient.addType(type,null);
dictionaryClient.addType(type, null);
dictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
@ -374,4 +375,60 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
assertThat(dictionary.getEntries().size()).isEqualTo(5);
}
@Test
public void testCreateAndDeleteLargeDictionary() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var type = CreateTypeValue.builder()
.type("dossier_redaction")
.label("Dossier Redactions")
.hexColor("#fcba03")
.rank(100)
.description("Something")
.hasDictionary(true)
.addToDictionaryAction(false)
.dossierTemplateId(dossierTemplate.getId())
.build();
var createdType = dictionaryClient.addType(type, null);
var entries = createDummyEntries(40_000);
dictionaryClient.addEntry(createdType.getType(), createdType.getDossierTemplateId(), entries, false, null, DictionaryEntryType.ENTRY);
var dictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
var actualEntries = dictionary.getEntries();
assertThat(actualEntries.size()).isEqualTo(entries.size());
assertThat(actualEntries).usingComparator(new ListContentWithoutOrderAndWithoutDuplicatesComparator<>()).isEqualTo(entries);
dictionaryClient.deleteEntries(type.getType(), dossierTemplate.getDossierTemplateId(), entries, null, null);
dictionary = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
actualEntries = dictionary.getEntries();
assertThat(actualEntries.size()).isEqualTo(0);
}
@SuppressWarnings("SameParameterValue")
private List<String> createDummyEntries(int numberOfEntries) {
return IntStream.range(0, numberOfEntries).mapToObj(i -> "someWord" + i).toList();
}
private static final class ListContentWithoutOrderAndWithoutDuplicatesComparator<T> implements Comparator<List<? extends T>> {
@SuppressWarnings("SuspiciousMethodCalls")
@Override
public int compare(List<? extends T> l1, List<? extends T> l2) {
return l1.containsAll(l2) ? 0 : -1;
}
}
}