Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3932 Performance update for dictionary operations

Browse Source
This commit is contained in:
Timo Bejan 2022-04-26 17:34:54 +03:00
parent 54a086891e
commit 1b488b1919
10 changed files with 186 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/entity/configuration/DictionaryEntryEntity.java
View File

@ -25,7 +25,6 @@ public class DictionaryEntryEntity implements BaseDictionaryEntry {
private long version;
@Column
private boolean deleted;
@JsonIgnore
@ManyToOne(fetch = FetchType.LAZY)
private TypeEntity type;

5
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DossierTemplateCloneService.java
View File

@ -7,6 +7,7 @@ import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@ -143,8 +144,8 @@ public class DossierTemplateCloneService {
clonedTypes.add(te);
for (DictionaryEntryType det : DictionaryEntryType.values()) {
var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null);
List<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toList());
entryPersistenceService.addEntry(te.getId(), entries, te.getVersion(), det);
Set<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet());
entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det);
}
}
return clonedTypes;

129
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java
View File

@ -11,13 +11,11 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@ -30,55 +28,6 @@ public class EntryPersistenceService {
private final FalsePositiveEntryRepository falsePositiveEntryRepository;
private final FalseRecommendationEntryRepository falseRecommendationEntryRepository;
public void addEntry(String typeId, List<String> entries, long version, DictionaryEntryType dictionaryEntryType) {
var type = typeRepository.getById(typeId);
switch (dictionaryEntryType) {
case ENTRY:
entries.stream().forEach(word -> {
DictionaryEntryEntity entry = new DictionaryEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
try {
entryRepository.save(entry);
} catch (DataIntegrityViolationException e) {
entryRepository.undeleteEntryIfDeleted(typeId, entry.getValue(), version);
}
});
break;
case FALSE_POSITIVE:
entries.stream().forEach(word -> {
DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
try {
falsePositiveEntryRepository.save(entry);
} catch (DataIntegrityViolationException e) {
falsePositiveEntryRepository.undeleteEntryIfDeleted(typeId, entry.getValue(), version);
}
});
break;
case FALSE_RECOMMENDATION:
entries.stream().forEach(word -> {
DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity();
entry.setVersion(version);
entry.setValue(word);
entry.setType(type);
try {
falseRecommendationEntryRepository.save(entry);
} catch (DataIntegrityViolationException e) {
falseRecommendationEntryRepository.undeleteEntryIfDeleted(typeId, entry.getValue(), version);
}
});
break;
}
}
@Transactional
public void deleteEntries(String typeId, List<String> values, long version,
DictionaryEntryType dictionaryEntryType) {
@ -98,17 +47,17 @@ public class EntryPersistenceService {
@Transactional
public void setVersion(String typeId, List<String> values, long version, DictionaryEntryType dictionaryEntryType) {
public void setVersion(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
entryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
falsePositiveEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.updateVersionWhereTypeIdAndValueIn(version, typeId, values);
falseRecommendationEntryRepository.updateVersionWhereTypeId(version, typeId);
break;
}
}
@ -127,4 +76,72 @@ public class EntryPersistenceService {
return null;
}
public void deleteAllEntriesForTypeId(String typeId, long version, DictionaryEntryType dictionaryEntryType) {
switch (dictionaryEntryType) {
case ENTRY:
entryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_POSITIVE:
falsePositiveEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
case FALSE_RECOMMENDATION:
falseRecommendationEntryRepository.deleteAllEntriesForTypeId(typeId, version);
break;
}
}
public void addEntries(String typeId, Set<String> entries, long version, DictionaryEntryType dictionaryEntryType) {
var type = typeRepository.getById(typeId);
switch (dictionaryEntryType) {
case ENTRY: {
var undeletedEntries = entryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
var entryEntities = entries.stream().map(e -> {
DictionaryEntryEntity entry = new DictionaryEntryEntity();
entry.setVersion(version);
entry.setValue(e);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
entryRepository.saveAll(entryEntities);
break;
}
case FALSE_POSITIVE: {
var undeletedEntries = falsePositiveEntryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
var entryEntities = entries.stream().map(e -> {
DictionaryFalsePositiveEntryEntity entry = new DictionaryFalsePositiveEntryEntity();
entry.setVersion(version);
entry.setValue(e);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
falsePositiveEntryRepository.saveAll(entryEntities);
break;
}
case FALSE_RECOMMENDATION: {
var undeletedEntries = falseRecommendationEntryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
var entryEntities = entries.stream().map(e -> {
DictionaryFalseRecommendationEntryEntity entry = new DictionaryFalseRecommendationEntryEntity();
entry.setVersion(version);
entry.setValue(e);
entry.setType(type);
return entry;
}).collect(Collectors.toList());
falseRecommendationEntryRepository.saveAll(entryEntities);
break;
}
}
}
}

18
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -13,24 +14,29 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Long> {
@Modifying
@Query("update DictionaryEntryEntity e set e.deleted = true , e.version = :version where e.type.id =:typeId and e.value in :values")
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id =:typeId and e.value in :values")
void deleteAllByTypeIdAndVersionAndValueIn(String typeId, long version, List<String> values);
@Modifying
@Query("update DictionaryEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values")
void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List<String> values);
@Query("update DictionaryEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
@Modifying
@Query("update DictionaryEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId")
@Query("update DictionaryEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId and e.deleted = false")
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
List<DictionaryEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
@Modifying
@Transactional
@Query("update DictionaryEntryEntity e set e.deleted = false, e.version = :version where e.type.id = :typeId and e.value = :value and e.deleted = true")
void undeleteEntryIfDeleted(String typeId, String value, long version);
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query(value = "update dictionary_entry set deleted = true, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

27
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java
View File

@ -1,14 +1,13 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import javax.transaction.Transactional;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import javax.transaction.Transactional;
import java.util.List;
import java.util.Set;
public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@ -18,21 +17,19 @@ public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFa
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values")
void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List<String> values);
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId")
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
@Query("update DictionaryFalsePositiveEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
List<DictionaryFalsePositiveEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
@Modifying
@Transactional
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = false, e.version = :version where e.type.id = :typeId and e.value = :value and e.deleted = true")
void undeleteEntryIfDeleted(String typeId, String value, long version);
@Query(value = "update dictionary_false_positive_entry set deleted = true, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

19
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -19,21 +20,19 @@ public interface FalseRecommendationEntryRepository extends JpaRepository<Dictio
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.value in :values")
void updateVersionWhereTypeIdAndValueIn(long version, String typeId, List<String> values);
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.type.id = :newTypeId, e.version = e.version + 1 where e.type.id =:typeId")
void updateTypeIdAndIncrementVersionByOne(String typeId, String newTypeId);
@Query("update DictionaryFalseRecommendationEntryEntity e set e.version = :version where e.type.id =:typeId and e.deleted = false")
void updateVersionWhereTypeId(long version, String typeId);
List<DictionaryFalseRecommendationEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
@Modifying
@Transactional
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true, e.version = :version where e.type.id = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = false, e.version = :version where e.type.id = :typeId and e.value = :value and e.deleted = true")
void undeleteEntryIfDeleted(String typeId, String value, long version);
@Query(value = "update dictionary_false_recommendation_entry set deleted = true, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

44
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/DictionaryController.java
View File

@ -80,23 +80,10 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
if (removeCurrent) {
List<String> existing = entryPersistenceService.getEntries(typeId, dictionaryEntryType, null)
.stream()
.filter(e -> !e.isDeleted())
.map(BaseDictionaryEntry::getValue)
.collect(toList());
List<String> removed = new ArrayList<>(existing);
removed.removeAll(cleanEntries);
List<String> added = new ArrayList<>(cleanEntries);
added.removeAll(existing);
entryPersistenceService.deleteEntries(typeId, removed, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntry(typeId, added, currentVersion + 1, dictionaryEntryType);
entryPersistenceService.deleteAllEntriesForTypeId(typeId,currentVersion+1,dictionaryEntryType);
entryPersistenceService.addEntries(typeId,cleanEntries, currentVersion+1, dictionaryEntryType);
} else {
entryPersistenceService.addEntry(typeId, new ArrayList<>(cleanEntries), currentVersion + 1, dictionaryEntryType);
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
}
dictionaryPersistenceService.incrementVersion(typeId);
@ -148,23 +135,10 @@ public class DictionaryController implements DictionaryResource {
.getRank() != typeValueRequest.getRank()) {
var currentVersion = getCurrentVersion(typeResult);
var entries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, entries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1, DictionaryEntryType.ENTRY);
var falsePositiveEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_POSITIVE, null), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, falsePositiveEntries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
var falseRecommendationEntries = convert(entryPersistenceService.getEntries(typeId, DictionaryEntryType.FALSE_RECOMMENDATION, null), DictionaryEntry.class);
entryPersistenceService.setVersion(typeId, falseRecommendationEntries.stream()
.filter(entry -> !entry.isDeleted())
.map(DictionaryEntry::getValue)
.collect(toList()), currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.setVersion(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.setVersion(typeId,currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
}
@ -215,9 +189,11 @@ public class DictionaryController implements DictionaryResource {
var currentVersion = getCurrentVersion(typeResult);
dictionaryPersistenceService.deleteType(typeId);
List<String> existing = entryPersistenceService.getEntries(typeId, DictionaryEntryType.ENTRY, null).stream().map(BaseDictionaryEntry::getValue).collect(toList());
entryPersistenceService.deleteEntries(typeId, existing, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.incrementVersion(typeId);
}

61
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/25-add-index-to-dictionary-entry-tables.yaml Normal file
View File

@ -0,0 +1,61 @@
databaseChangeLog:
- changeSet:
id: add-unique-constraint-for-dictionary
author: timo
changes:
- dropIndex:
- indexName: dictionary_entry_version_idx
- dropIndex:
- indexName: dictionary_entry_type_id_idx
- createIndex:
columns:
- column:
name: type_id
- column:
name: value
indexName: dictionary_entry_type_id_value_idx
tableName: dictionary_entry
- createIndex:
columns:
- column:
name: type_id
- column:
name: version
indexName: dictionary_entry_type_id_version_idx
tableName: dictionary_entry
- createIndex:
columns:
- column:
name: type_id
- column:
name: value
indexName: dictionary_false_positive_entry_type_id_value_idx
tableName: dictionary_false_positive_entry
- createIndex:
columns:
- column:
name: type_id
- column:
name: version
indexName: dictionary_false_positive_entry_type_id_version_idx
tableName: dictionary_false_positive_entry
- createIndex:
columns:
- column:
name: type_id
- column:
name: value
indexName: dictionary_false_recommendation_entry_type_id_value_idx
tableName: dictionary_false_recommendation_entry
- createIndex:
columns:
- column:
name: type_id
- column:
name: version
indexName: dictionary_false_recommendation_entry_type_id_version_idx
tableName: dictionary_false_recommendation_entry

4
persistence-service-v1/persistence-service-server-v1/src/main/resources/db/changelog/db.changelog-master.yaml
View File

@ -58,4 +58,6 @@ databaseChangeLog:
- include:
file: db/changelog/sql/24.0-clean-up-duplicate-dictionary-entries.sql
- include:
file: db/changelog/24-add-unique-constraint-for-dictionary.yaml
file: db/changelog/24-add-unique-constraint-for-dictionary.yaml
- include:
file: db/changelog/25-add-index-to-dictionary-entry-tables.yaml

12
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/FilePerformanceTest.java
View File

@ -103,17 +103,11 @@ public class FilePerformanceTest extends AbstractPersistenceServerServiceTest {
de.setType(te);
de.setDeleted(false);
de.setVersion(1);
de.setValue("Dictionary Entry" + i);
de.setValue("Dictionary Entry" +i+"/"+ j);
des.add(de);
try {
entryRepository.save(de);
} catch (DataIntegrityViolationException e) {
log.info("Duplicate Type: {}", te.getType());
entryRepository.undeleteEntryIfDeleted(te.getId(), de.getValue(), de.getVersion());
}
}
}
entryRepository.saveAll(des);
log.info("Created Type: {}", te.getType());
}