RED-6467: Backport of change from 4.0.

* Implemented undeletion of dictionary entries by running a native query in chunks.
This avoids a limitation in the JDBC driver.
* Changed unique name check to not use Exceptions to prevent transaction rollbacks
This commit is contained in:
Viktor Seifert 2023-04-21 16:53:04 +02:00
parent 9742afe175
commit cf0c2d6100
17 changed files with 259 additions and 65 deletions

View File

@ -17,8 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionarySummaryResponse;
import lombok.RequiredArgsConstructor;

View File

@ -96,13 +96,19 @@ public class DossierTemplatePersistenceService {
}
@Transactional
public void validateDossierTemplateNameIsUnique(String templateName) {
getAllDossierTemplates().forEach(existing -> {
if (existing.getName().equals(templateName)) {
throw new ConflictException("DossierTemplate name must be unique");
}
});
if (isDossierTemplateNameNotUnique(templateName)) {
throw new ConflictException("DossierTemplate name must be unique");
}
}
@Transactional
public boolean isDossierTemplateNameNotUnique(String templateName) {
return dossierTemplateRepository.existsByName(templateName);
}

View File

@ -12,10 +12,10 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;

View File

@ -17,4 +17,7 @@ public interface DossierTemplateRepository extends JpaRepository<DossierTemplate
@Query("select d from DossierTemplateEntity d where d.id = :dossierTemplateId and d.softDeleteTime is null")
Optional<DossierTemplateEntity> findByIdAndNotDeleted(String dossierTemplateId);
}
boolean existsByName(String name);
}

View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Long> {
public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<DictionaryEntryEntity, Long> {
@Modifying
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
@ -31,18 +30,11 @@ public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Lo
List<DictionaryEntryEntity> findByTypeIdAndVersionGreaterThan(String typeId, long version);
@Modifying
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "update dictionary_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface EntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

View File

@ -0,0 +1,26 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Repository
public class EntryRepositoryImpl implements EntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_entry");
}
}

View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
public interface FalsePositiveEntryRepository extends FalsePositiveEntryRepositoryCustom, JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
@ -31,13 +30,6 @@ public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFa
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query(value = "update dictionary_false_positive_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface FalsePositiveEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

View File

@ -0,0 +1,26 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Repository
class FalsePositiveEntryRepositoryImpl implements FalsePositiveEntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_positive_entry");
}
}

View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
public interface FalseRecommendationEntryRepository extends JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
public interface FalseRecommendationEntryRepository extends FalseRecommendationEntryRepositoryCustom, JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
@ -32,12 +31,6 @@ public interface FalseRecommendationEntryRepository extends JpaRepository<Dictio
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query(value = "update dictionary_false_recommendation_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_recommendation_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_recommendation_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface FalseRecommendationEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

View File

@ -0,0 +1,26 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Repository
class FalseRecommendationEntryRepositoryImpl implements FalseRecommendationEntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_recommendation_entry");
}
}

View File

@ -0,0 +1,109 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.transaction.Transactional;
import org.springframework.stereotype.Component;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@Component
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
class QueryExecutor {
private static final String FETCH_ENTRY_VALUES_QUERY = """
select value from ::tableName::
where type_id = :typeId and value in (:entries)""";
private static final String UPDATE_ENTRIES_QUERY = """
update ::tableName::
set deleted = false, version = :version
where type_id = :typeId and value in (:entries)""";
// Currently (2023-04-13) there is a limitation in the Postgres JDBC driver, that limits the number of elements in a "IN" clause
// to the max value of a 'short'. We subtract a small value to be on the safe side, since it is unclear what contributes
// to the number of elements, only the elements or parentheses etc.
private static final int ELEMENT_CHUNK_SIZE = Short.MAX_VALUE - 10;
EntityManager entityManager;
@Transactional
public LinkedList<String> runUndeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
var results = new LinkedList<String>();
var entryList = new ArrayList<>(entries);
for (int fromIndex = 0, toIndex = ELEMENT_CHUNK_SIZE; ; ) {
toIndex = Math.min(toIndex, entryList.size());
if (fromIndex >= entryList.size()) {
break;
}
var values = entryList.subList(fromIndex, toIndex);
var entryValues = executeFetchValuesQuery(typeId, tableName, values);
results.addAll(entryValues);
executeUpdateQuery(typeId, version, tableName, values);
fromIndex += ELEMENT_CHUNK_SIZE;
toIndex += ELEMENT_CHUNK_SIZE;
}
return results;
}
private void executeUpdateQuery(String typeId, long version, String tableName, List<String> values) {
String updateSql = getUpdateEntriesQuery(tableName);
Query updateEntriesQuery = entityManager.createNativeQuery(updateSql);
updateEntriesQuery.setParameter("typeId", typeId);
updateEntriesQuery.setParameter("version", version);
updateEntriesQuery.setParameter("entries", values);
updateEntriesQuery.executeUpdate();
}
// The call to query.getResultList returns an untyped list, there is no way around that.
// So we suppress the warning.
// CAUTION: Make sure that the query actually returns a list of Strings.
@SuppressWarnings("unchecked")
private List<String> executeFetchValuesQuery(String typeId, String tableName, List<String> values) {
String fetchSql = getFetchEntryValuesQuery(tableName);
Query fetchEntryValuesQuery = entityManager.createNativeQuery(fetchSql);
fetchEntryValuesQuery.setParameter("typeId", typeId);
fetchEntryValuesQuery.setParameter("entries", values);
return fetchEntryValuesQuery.getResultList();
}
private String getFetchEntryValuesQuery(String tableName) {
return FETCH_ENTRY_VALUES_QUERY.replace("::tableName::", tableName);
}
private String getUpdateEntriesQuery(String tableName) {
return UPDATE_ENTRIES_QUERY.replace("::tableName::", tableName);
}
}

View File

@ -24,7 +24,6 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
@ -36,6 +35,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.peristence.v1.server.utils.FileUtils;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
@ -376,22 +376,16 @@ public class DossierTemplateImportService {
private void validateDossierTemplateName(DossierTemplate dossierTemplateMeta) {
boolean cond = true;
int index = 0;
int nameSuffix = 0;
String dossierTemplateName = dossierTemplateMeta.getName();
do {
try {
dossierTemplatePersistenceService.validateDossierTemplateNameIsUnique(dossierTemplateMeta.getName());
cond = false;
} catch (ConflictException e) {
if (index == 0) {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName);
} else {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + index);
}
index++;
while (dossierTemplatePersistenceService.isDossierTemplateNameNotUnique(dossierTemplateMeta.getName())) {
if (nameSuffix == 0) {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName);
} else {
dossierTemplateMeta.setName("Copy of " + dossierTemplateName + " - " + nameSuffix);
}
} while (cond);
nameSuffix++;
}
}
@ -449,10 +443,8 @@ public class DossierTemplateImportService {
dictionaryPersistenceService.incrementVersion(typeId);
typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
});
Set<String> typesToRemove = currentTypes.stream()
.filter(t -> !t.isDeleted()) // remove the ones already soft deleted
.map(TypeEntity::getId)
.filter(t -> !typeIdsAdded.contains(t)) // exclude the type ids already added from the import
Set<String> typesToRemove = currentTypes.stream().filter(t -> !t.isDeleted()) // remove the ones already soft deleted
.map(TypeEntity::getId).filter(t -> !typeIdsAdded.contains(t)) // exclude the type ids already added from the import
.filter(t -> !currentTypesIdSystemManaged.contains(t)) // exclude the types system managed
.collect(Collectors.toSet());
typesToRemove.forEach(dictionaryService::deleteType);
@ -568,7 +560,7 @@ public class DossierTemplateImportService {
double compressionRatio = (float) totalSizeEntry / ze.getCompressedSize();
if (compressionRatio > settings.getCompressionThresholdRatio()) {
log.debug("zip entry: " + ze.getName() + " - totalSizeEntry: " + totalSizeEntry + " ze.getCompressedSize(): " + ze.getCompressedSize() + " compressionRatio: " + compressionRatio);
log.debug("zip entry: " + ze.getName() + " - totalSizeEntry: " + totalSizeEntry + " ze.getCompressedSize(): " + ze.getCompressedSize() + " compressionRatio: " + compressionRatio);
// ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack
throw new BadRequestException("ZIP-Bomb detected (compressionRatio).");
}

View File

@ -15,7 +15,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;

View File

@ -54,9 +54,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
@ -76,6 +73,9 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ViewedPagesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.persistence.service.v1.api.model.multitenancy.TenantRequest;
@ -182,7 +182,6 @@ public abstract class AbstractPersistenceServerServiceTest {
protected PrometheusMeterRegistry prometheusMeterRegistry;
@Before
public void setupOptimize() {