Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-6467: Implemented undeletion of dictionary entries by running a native query in chunks.

Browse Source 
This avoids a limitation in the JDBC driver.
This commit is contained in:
Viktor Seifert 2023-04-13 15:34:07 +02:00
parent 62dbe67d63
commit c4648fa9fb
16 changed files with 225 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/DictionaryPersistenceService.java
View File

@ -17,8 +17,8 @@ import com.iqser.red.service.persistence.management.v1.processor.exception.BadRe
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummaryResponse;
import lombok.RequiredArgsConstructor;

84
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/EntryPersistenceService.java
View File

@ -5,7 +5,6 @@ import java.util.Set;
import java.util.stream.Collectors;
import javax.persistence.EntityManager;
import javax.persistence.criteria.CriteriaBuilder;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
@ -14,10 +13,10 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@ -112,9 +111,7 @@ public class EntryPersistenceService {
switch (dictionaryEntryType) {
case ENTRY -> {
// TODO Remove all but one implementation for undeletion, once it is clear which one works
//var undeletedEntries = entryRepository.undeleteEntries(typeId, entries, version);
var undeletedEntries = undeleteEntries(typeId, entries, version);
var undeletedEntries = entryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
@ -130,7 +127,7 @@ public class EntryPersistenceService {
jdbcWriteUtils.saveBatch(entryEntities);
}
case FALSE_POSITIVE -> {
var undeletedEntries = undeleteFalsePositives(typeId, entries, version);
var undeletedEntries = falsePositiveEntryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
var entryEntities = entries.stream().map(e -> {
@ -144,7 +141,7 @@ public class EntryPersistenceService {
jdbcWriteUtils.saveBatch(entryEntities);
}
case FALSE_RECOMMENDATION -> {
var undeletedEntries = undeleteFalseRecommendations(typeId, entries, version);
var undeletedEntries = falseRecommendationEntryRepository.undeleteEntries(typeId, entries, version);
undeletedEntries.forEach(entries::remove);
var entryEntities = entries.stream().map(e -> {
@ -161,75 +158,6 @@ public class EntryPersistenceService {
}
private List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder();
var undeleteEntriesQuery = criteriaBuilder.createCriteriaUpdate(DictionaryEntryEntity.class);
var entryRoot = undeleteEntriesQuery.from(DictionaryEntryEntity.class);
var typeIdPred = criteriaBuilder.equal(entryRoot.get("typeId"), typeId);
var entriesPred = entryRoot.get("value").in(entries);
undeleteEntriesQuery.where(typeIdPred, entriesPred);
undeleteEntriesQuery.set("version", version);
undeleteEntriesQuery.set("deleted", false);
int updated = entityManager.createQuery(undeleteEntriesQuery).executeUpdate();
var existingEntriesQuery = criteriaBuilder.createQuery(DictionaryEntryEntity.class);
entryRoot = existingEntriesQuery.from(DictionaryEntryEntity.class);
existingEntriesQuery.where(typeIdPred, criteriaBuilder.equal(entryRoot.get("deleted"), false));
return entityManager.createQuery(existingEntriesQuery).getResultStream().map(DictionaryEntryEntity::getValue).toList();
}
private List<String> undeleteFalsePositives(String typeId, Set<String> entries, long version) {
CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder();
var undeleteEntriesQuery = criteriaBuilder.createCriteriaUpdate(DictionaryFalsePositiveEntryEntity.class);
var entryRoot = undeleteEntriesQuery.from(DictionaryFalsePositiveEntryEntity.class);
var typeIdPred = criteriaBuilder.equal(entryRoot.get("typeId"), typeId);
var entriesPred = entryRoot.get("value").in(entries);
undeleteEntriesQuery.where(typeIdPred, entriesPred);
undeleteEntriesQuery.set("version", version);
undeleteEntriesQuery.set("deleted", false);
int updated = entityManager.createQuery(undeleteEntriesQuery).executeUpdate();
var existingEntriesQuery = criteriaBuilder.createQuery(DictionaryFalsePositiveEntryEntity.class);
entryRoot = existingEntriesQuery.from(DictionaryFalsePositiveEntryEntity.class);
existingEntriesQuery.where(typeIdPred, criteriaBuilder.equal(entryRoot.get("deleted"), false));
return entityManager.createQuery(existingEntriesQuery).getResultStream().map(DictionaryFalsePositiveEntryEntity::getValue).toList();
}
private List<String> undeleteFalseRecommendations(String typeId, Set<String> entries, long version) {
CriteriaBuilder criteriaBuilder = entityManager.getCriteriaBuilder();
var undeleteEntriesQuery = criteriaBuilder.createCriteriaUpdate(DictionaryFalseRecommendationEntryEntity.class);
var entryRoot = undeleteEntriesQuery.from(DictionaryFalseRecommendationEntryEntity.class);
var typeIdPred = criteriaBuilder.equal(entryRoot.get("typeId"), typeId);
var entriesPred = entryRoot.get("value").in(entries);
undeleteEntriesQuery.where(typeIdPred, entriesPred);
undeleteEntriesQuery.set("version", version);
undeleteEntriesQuery.set("deleted", false);
int updated = entityManager.createQuery(undeleteEntriesQuery).executeUpdate();
var existingEntriesQuery = criteriaBuilder.createQuery(DictionaryFalseRecommendationEntryEntity.class);
entryRoot = existingEntriesQuery.from(DictionaryFalseRecommendationEntryEntity.class);
existingEntriesQuery.where(typeIdPred, criteriaBuilder.equal(entryRoot.get("deleted"), false));
return entityManager.createQuery(existingEntriesQuery).getResultStream().map(DictionaryFalseRecommendationEntryEntity::getValue).toList();
}
public void cloneEntries(String originalTypeId, String newTypeId) {
entryRepository.cloneEntries(originalTypeId, newTypeId);

9
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/EntryRepository.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepository.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Long> {
public interface EntryRepository extends EntryRepositoryCustom, JpaRepository<DictionaryEntryEntity, Long> {
@Modifying
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId and e.value in :values")
@ -36,10 +35,6 @@ public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Lo
@Query("update DictionaryEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
List<DictionaryEntryEntity> findAllByTypeIdAndValueIn(String typeId, Set<String> entries);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryCustom.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface EntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

26
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/EntryRepositoryImpl.java Normal file
View File

@ -0,0 +1,26 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Repository
public class EntryRepositoryImpl implements EntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_entry");
}
}

12
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalsePositiveEntryRepository.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepository.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalsePositiveEntryEntity;
public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
public interface FalsePositiveEntryRepository extends FalsePositiveEntryRepositoryCustom, JpaRepository<DictionaryFalsePositiveEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
@ -31,13 +30,6 @@ public interface FalsePositiveEntryRepository extends JpaRepository<DictionaryFa
@Query("update DictionaryFalsePositiveEntryEntity e set e.deleted = true, e.version = :version where e.typeId = :typeId")
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query(value = "update dictionary_false_positive_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_positive_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_positive_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryCustom.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface FalsePositiveEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

23
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalsePositiveEntryRepositoryImpl.java Normal file
View File

@ -0,0 +1,23 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Repository
class FalsePositiveEntryRepositoryImpl implements FalsePositiveEntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_positive_entry");
}
}

11
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/FalseRecommendationEntryRepository.java → persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepository.java
View File

@ -1,7 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository;
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import javax.transaction.Transactional;
@ -11,7 +10,7 @@ import org.springframework.data.jpa.repository.Query;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryFalseRecommendationEntryEntity;
public interface FalseRecommendationEntryRepository extends JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
public interface FalseRecommendationEntryRepository extends FalseRecommendationEntryRepositoryCustom, JpaRepository<DictionaryFalseRecommendationEntryEntity, Long> {
@Modifying
@Query("update DictionaryFalseRecommendationEntryEntity e set e.deleted = true , e.version = :version where e.typeId = :typeId and e.value in :values")
@ -32,12 +31,6 @@ public interface FalseRecommendationEntryRepository extends JpaRepository<Dictio
void deleteAllEntriesForTypeId(String typeId, long version);
@Modifying
@Transactional
@Query(value = "update dictionary_false_recommendation_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_false_recommendation_entry (value, version, deleted, type_id) " + " select value, 1, false, :newTypeId from dictionary_false_recommendation_entry where type_id = :originalTypeId and deleted = false", nativeQuery = true)

10
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryCustom.java Normal file
View File

@ -0,0 +1,10 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
public interface FalseRecommendationEntryRepositoryCustom {
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
}

23
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/FalseRecommendationEntryRepositoryImpl.java Normal file
View File

@ -0,0 +1,23 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Repository;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Repository
class FalseRecommendationEntryRepositoryImpl implements FalseRecommendationEntryRepositoryCustom {
QueryExecutor queryExecutor;
@Override
public List<String> undeleteEntries(String typeId, Set<String> entries, long version) {
return queryExecutor.runUndeleteQueryInBatches(typeId, entries, version, "dictionary_false_recommendation_entry");
}
}

73
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/persistence/repository/dictionaryentry/QueryExecutor.java Normal file
View File

@ -0,0 +1,73 @@
package com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Set;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import org.springframework.stereotype.Component;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
@RequiredArgsConstructor
@Component
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
class QueryExecutor {
private static final String UNDELETE_ENTRIES_QUERY = """
update ::tableName::
set deleted = false, version = :version
where type_id = :typeId and value in (:entries) returning value""";
// Currently (2023-04-13) there is a limitation in the Postgres JDBC driver, that limits the number of elements in a "IN" clause
// to the max value of a 'short'. We subtract a small value to be on the safe side, since it is unclear what contributes
// to the number of elements, only the elements or parentheses etc.
private static final int ELEMENT_CHUNK_SIZE = Short.MAX_VALUE - 10;
EntityManager entityManager;
// The call to query.getResultList returns an untyped list, there is no way around that.
// So we suppress the warning.
// CAUTION: Make sure that the query actually returns a list of Strings.
@SuppressWarnings("unchecked")
public LinkedList<String> runUndeleteQueryInBatches(String typeId, Set<String> entries, long version, String tableName) {
String sqlString = getUndeleteEntriesQuery(tableName);
Query query = entityManager.createNativeQuery(sqlString);
query.setParameter("typeId", typeId);
query.setParameter("version", version);
var results = new LinkedList<String>();
var entryList = new ArrayList<>(entries);
for (int fromIndex = 0, toIndex = ELEMENT_CHUNK_SIZE; ; ) {
toIndex = Math.min(toIndex, entryList.size());
if (fromIndex >= entryList.size()) {
break;
}
query.setParameter("entries", entryList.subList(fromIndex, toIndex));
results.addAll(query.getResultList());
fromIndex += ELEMENT_CHUNK_SIZE;
toIndex += ELEMENT_CHUNK_SIZE;
}
return results;
}
private String getUndeleteEntriesQuery(String tableName) {
return UNDELETE_ENTRIES_QUERY.replace("::tableName::", tableName);
}
}

38
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DossierTemplateStatsTest.java
View File

@ -8,7 +8,6 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@ -37,10 +36,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.PageExclusi
import com.iqser.red.service.persistence.service.v1.api.shared.model.PageRange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplateDictionaryStats;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionarySummary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTest {
@ -103,7 +100,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.description("Something")
.addToDictionaryAction(false)
.dossierTemplateId(dossier.getDossierTemplateId())
.build(),null);
.build(), null);
assertThat(addedType1).isNotNull();
@ -118,13 +115,13 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.addToDictionaryAction(false)
.dossierTemplateId(dossierTemplate2.getId())
.hasDictionary(true)
.build(),null);
.build(), null);
assertThat(addedType).isNotNull();
var entries1 = new ArrayList<String>();
entries1.add("entry1");
entries1.add("entry2");
dictionaryClient.addEntry(addedType.getType(),addedType.getDossierTemplateId(), entries1, false, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(addedType.getType(), addedType.getDossierTemplateId(), entries1, false, null, DictionaryEntryType.ENTRY);
var dictionary = dictionaryClient.getDictionaryForType(addedType.getType(), addedType.getDossierTemplateId(), null);
assertThat(dictionary.getEntries().size()).isEqualTo(entries1.size());
@ -138,13 +135,12 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.addToDictionaryAction(false)
.dossierTemplateId(dossierTemplate2.getId())
.hasDictionary(true)
.build(),null);
.build(), null);
var entries2 = new ArrayList<String>();
entries2.add("entry1");
entries2.add("entry2");
entries2.add("entry3");
dictionaryClient.addEntry(addedType2.getType(),addedType2.getDossierTemplateId(), entries2, false, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(addedType2.getType(), addedType2.getDossierTemplateId(), entries2, false, null, DictionaryEntryType.ENTRY);
dictionary = dictionaryClient.getDictionaryForType(addedType2.getType(), addedType.getDossierTemplateId(), null);
assertThat(dictionary.getEntries().size()).isEqualTo(entries2.size());
@ -159,13 +155,13 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.addToDictionaryAction(false)
.dossierTemplateId(dossierTemplate3.getId())
.hasDictionary(true)
.build(),null);
.build(), null);
assertThat(addedType3).isNotNull();
var entries3 = new ArrayList<String>();
entries3.add("entry1");
entries3.add("entry2");
dictionaryClient.addEntry(addedType3.getType(),addedType3.getDossierTemplateId(), entries3, false, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(addedType3.getType(), addedType3.getDossierTemplateId(), entries3, false, null, DictionaryEntryType.ENTRY);
dictionary = dictionaryClient.getDictionaryForType(addedType3.getType(), addedType3.getDossierTemplateId(), null);
assertThat(dictionary.getEntries().size()).isEqualTo(entries3.size());
@ -183,7 +179,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.addToDictionaryAction(false)
.dossierTemplateId(dossierTemplate5.getId())
.hasDictionary(true)
.build(),null);
.build(), null);
assertThat(addedType5).isNotNull();
Set<String> dossierTemplateIds = new HashSet<>();
@ -218,7 +214,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var entries22 = new ArrayList<String>();
entries22.add(entries2.get(2));
dictionaryClient.deleteEntries(addedType2.getType(), addedType2.getDossierTemplateId(),entries22,null, DictionaryEntryType.ENTRY);
dictionaryClient.deleteEntries(addedType2.getType(), addedType2.getDossierTemplateId(), entries22, null, DictionaryEntryType.ENTRY);
dossierTemplateStatsList = dossierTemplateStatsClient.getDossierTemplateStats(dossierTemplateIds);
assertThat(dossierTemplateStatsList.size()).isEqualTo(dossierTemplateIds.size());
@ -235,7 +231,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
entries23.add(entries2.get(0));
entries23.add(entries2.get(1));
dictionaryClient.deleteEntries(addedType2.getType(),addedType2.getDossierTemplateId(), entries23, null,DictionaryEntryType.ENTRY);
dictionaryClient.deleteEntries(addedType2.getType(), addedType2.getDossierTemplateId(), entries23, null, DictionaryEntryType.ENTRY);
dictionary = dictionaryClient.getDictionaryForType(addedType2.getType(), addedType.getDossierTemplateId(), null);
var entries23loaded = dictionary.getEntries();
@ -277,7 +273,12 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
for (int i = 0; i < 2; i++) {
var template = dossierTemplateTesterAndProvider.provideTestTemplate("test template: " + i);
var status = dossierStatusClient.createOrUpdateDossierStatus(DossierStatusRequest.builder().dossierTemplateId(template.getId()).name("test").color("#cccccc").rank(100).build());
var status = dossierStatusClient.createOrUpdateDossierStatus(DossierStatusRequest.builder()
.dossierTemplateId(template.getId())
.name("test")
.color("#cccccc")
.rank(100)
.build());
for (int j = 0; j < 8; j++) {
var dossier = dossierTesterAndProvider.provideTestDossierQuick(template, "test dossier: " + j + " - " + i, j % 2 == 0 ? status : null);
@ -318,7 +319,12 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
var template = dossierTemplateTesterAndProvider.provideTestTemplate("test template inactive: ");
var status = dossierStatusClient.createOrUpdateDossierStatus(DossierStatusRequest.builder().dossierTemplateId(template.getId()).name("test").color("#cccccc").rank(100).build());
var status = dossierStatusClient.createOrUpdateDossierStatus(DossierStatusRequest.builder()
.dossierTemplateId(template.getId())
.name("test")
.color("#cccccc")
.rank(100)
.build());
var dossier = dossierTesterAndProvider.provideTestDossierQuick(template, "test dossier: ", status);
dossierClient.deleteDossier(dossier.getId());

15
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/EntityPerformanceTest.java
View File

@ -15,7 +15,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTem
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.jdbc.JDBCWriteUtils;
import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
@ -46,6 +46,7 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
@Test
public void testAddToDictionary() {
TenantContext.setTenantId("redaction");
var template = dossierTemplateTesterAndProvider.provideTestTemplate("test");
var type = typeProvider.testAndProvideType(template);
@ -54,29 +55,29 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
var tenKEntries = generateEntries(10000);
long t1 = System.currentTimeMillis();
dictionaryClient.addEntry(type.getType(),type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.ENTRY);
long t2 = System.currentTimeMillis();
log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
t1 = System.currentTimeMillis();
dictionaryClient.addEntry(type.getType(),type.getDossierTemplateId(), tenKEntries, true, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), tenKEntries, true, null, DictionaryEntryType.ENTRY);
t2 = System.currentTimeMillis();
log.info("Add Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
t1 = System.currentTimeMillis();
dictionaryClient.addEntry(type.getType(),type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.ENTRY);
t2 = System.currentTimeMillis();
log.info("Update Time: {}ms counting: {} entries", (t2 - t1), entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(), 0).size());
dictionaryClient.addEntry(type.getType(),type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryClient.addEntry(type.getType(),type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.FALSE_POSITIVE);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), fiveKEntries, true, null, DictionaryEntryType.FALSE_POSITIVE);
t1 = System.currentTimeMillis();
var cloned = dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
t2 = System.currentTimeMillis();
log.info("Clone Time: {}", (t2 - t1));
var types = dictionaryClient.getAllTypes(cloned.getId(), null,false).getTypes();
var types = dictionaryClient.getAllTypes(cloned.getId(), null, false).getTypes();
assertThat(types.size()).isEqualTo(1);
for (var clonedType : types) {

14
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/performance/PerformanceTestService.java
View File

@ -19,7 +19,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierStatsClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.integration.client.WatermarkClient;
import com.iqser.red.service.peristence.v1.server.integration.service.UserProvider;
import com.iqser.red.service.persistence.management.v1.processor.acl.custom.dossier.DossierACLService;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.DictionaryEntryEntity;
@ -32,21 +31,18 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileEntity;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.TypeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.WatermarkRepository;
import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.WatermarkOrientation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileAttributeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.WorkflowStatus;
import liquibase.pro.packaged.A;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@ -89,8 +85,7 @@ public class PerformanceTestService {
@Transactional
public void doSetup(){
public void doSetup() {
long start = System.currentTimeMillis();
@ -170,8 +165,8 @@ public class PerformanceTestService {
log.info("Created Dossier Template Data ...");
var user = userProvider.getUserId();
SecurityContextHolder.setContext(new TransientSecurityContext(new TestingAuthenticationToken(user,"secret")));
var user = userProvider.getUserId();
SecurityContextHolder.setContext(new TransientSecurityContext(new TestingAuthenticationToken(user, "secret")));
for (int i = 0; i < DOSSIER_COUNT; i++) {
@ -247,7 +242,6 @@ public class PerformanceTestService {
}
private Set<Integer> getExcludedPages() {
Set<Integer> pages = new HashSet<>();

6
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -63,9 +63,6 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DownloadStatusRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributeConfigRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesGeneralConfigurationRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileAttributesRepository;
@ -85,6 +82,9 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.LegalBasisChangeRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.ManualRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.annotationentity.RemoveRedactionRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.EntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalsePositiveEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.dictionaryentry.FalseRecommendationEntryRepository;
import com.iqser.red.service.persistence.management.v1.processor.service.redactionlog.RedactionLogMergeService;
import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter;
import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext;