RED-3942 - Improve clone performance

This commit is contained in:
Timo Bejan 2022-05-23 17:38:35 +03:00
parent 5912517849
commit a2864b4e1c
8 changed files with 174 additions and 88 deletions

View File

@ -1,6 +1,24 @@
package com.iqser.red.service.persistence.management.v1.processor.service;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.time.OffsetDateTime;
@ -11,35 +29,7 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.*;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.FileAttributeConfigEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileAttributeConfigPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.LegalBasisMappingPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.RulesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.DossierTemplateRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DossierTemplateStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import static com.iqser.red.service.persistence.management.v1.processor.utils.MagicConverter.convert;
@Slf4j
@Service
@ -67,6 +57,7 @@ public class DossierTemplateCloneService {
DossierTemplateEntity clonedDossierTemplate = new DossierTemplateEntity();
long start=System.currentTimeMillis();
dossierTemplateRepository.findById(dossierTemplateId).ifPresentOrElse((dossierTemplate) -> {
OffsetDateTime now = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
@ -92,7 +83,10 @@ public class DossierTemplateCloneService {
dossierTemplateRepository.save(clonedDossierTemplate);
//set dictionaries
long t1 = System.currentTimeMillis();
clonedDossierTemplate.setDossierTypes(cloneDictionariesWithEntries(dossierTemplate.getId(), clonedDossierTemplate.getId()));
long t2 = System.currentTimeMillis();
log.warn("Time to clone entries: {}", (t2 - t1));
//set dossier attributes
cloneDossierAttributesConfig(dossierTemplate.getId(), clonedDossierTemplate.getId());
@ -119,6 +113,9 @@ public class DossierTemplateCloneService {
}, () -> {
throw new NotFoundException(String.format(dossierTemplatePersistenceService.DOSSIER_TEMPLATE_NOT_FOUND_MESSAGE, dossierTemplateId));
});
long end = System.currentTimeMillis();
System.out.println("END TIME: "+(end-start));
log.warn("End Time {}",end-start);
return clonedDossierTemplate;
}
@ -138,16 +135,18 @@ public class DossierTemplateCloneService {
private List<TypeEntity> cloneDictionariesWithEntries(String dossierTemplateId, String clonedDossierTemplateId) {
List<TypeEntity> clonedTypes = new ArrayList<>();
var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId,false);
var types = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
for (TypeEntity t : types) {
TypeEntity te = dictionaryPersistenceService.addType(t.getType(), clonedDossierTemplateId, t.getHexColor(), t.getRecommendationHexColor(), t.getRank(), t.isHint(), t.isCaseInsensitive(), t.isRecommendation(), t.getDescription(), t.isAddToDictionaryAction(), t.getLabel(), null, t.isHasDictionary(), t.isSystemManaged(), t.isAutoHideSkipped());
te.setDossierTemplateId(clonedDossierTemplateId);
clonedTypes.add(te);
for (DictionaryEntryType det : DictionaryEntryType.values()) {
var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null);
Set<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet());
entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det);
}
// for (DictionaryEntryType det : DictionaryEntryType.values()) {
// var baseDictionaryEntries = entryPersistenceService.getEntries(t.getId(), det, null);
// Set<String> entries = baseDictionaryEntries.stream().map(BaseDictionaryEntry::getValue).collect(Collectors.toSet());
// entryPersistenceService.addEntries(te.getId(), entries, te.getVersion(), det);
// }
entryPersistenceService.cloneEntries(dossierTemplateId,clonedDossierTemplateId);
}
return clonedTypes;
}
@ -172,13 +171,17 @@ public class DossierTemplateCloneService {
private void cloneFileAttributesGeneralConfig(String dossierTemplateId, String clonedDossierTemplateId) {
var fileAttributesGeneralConfig = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossierTemplateId);
var fagc = FileAttributesGeneralConfigurationEntity.builder()
.dossierTemplateId(clonedDossierTemplateId)
.delimiter(fileAttributesGeneralConfig.getDelimiter())
.filenameMappingColumnHeaderName(fileAttributesGeneralConfig.getFilenameMappingColumnHeaderName())
.build();
fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(clonedDossierTemplateId, fagc);
try {
var fileAttributesGeneralConfig = fileAttributeConfigPersistenceService.getFileAttributesGeneralConfiguration(dossierTemplateId);
var fagc = FileAttributesGeneralConfigurationEntity.builder()
.dossierTemplateId(clonedDossierTemplateId)
.delimiter(fileAttributesGeneralConfig.getDelimiter())
.filenameMappingColumnHeaderName(fileAttributesGeneralConfig.getFilenameMappingColumnHeaderName())
.build();
fileAttributeConfigPersistenceService.setFileAttributesGeneralConfig(clonedDossierTemplateId, fagc);
}catch (NotFoundException e){
log.debug("No file attribute config found for cloning {}", dossierTemplateId);
}
}
@ -225,17 +228,21 @@ public class DossierTemplateCloneService {
private void cloneWatermark(String dossierTemplateId, String clonedDossierTemplateId) {
var watermark = watermarkService.getWatermark(dossierTemplateId);
WatermarkEntity we = WatermarkEntity.builder()
.dossierTemplateId(clonedDossierTemplateId)
.text(watermark.getText())
.hexColor(watermark.getHexColor())
.opacity(watermark.getOpacity())
.fontType(watermark.getFontType())
.fontSize(watermark.getFontSize())
.orientation(watermark.getOrientation())
.build();
watermarkService.saveWatermark(clonedDossierTemplateId, we);
try {
var watermark = watermarkService.getWatermark(dossierTemplateId);
WatermarkEntity we = WatermarkEntity.builder()
.dossierTemplateId(clonedDossierTemplateId)
.text(watermark.getText())
.hexColor(watermark.getHexColor())
.opacity(watermark.getOpacity())
.fontType(watermark.getFontType())
.fontSize(watermark.getFontSize())
.orientation(watermark.getOrientation())
.build();
watermarkService.saveWatermark(clonedDossierTemplateId, we);
}catch (NotFoundException e){
log.debug("No watermark config found for cloning {}", dossierTemplateId);
}
}

View File

@ -144,4 +144,10 @@ public class EntryPersistenceService {
}
}
}
public void cloneEntries(String dossierTemplateId, String clonedDossierTemplateId) {
entryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
// falseRecommendationEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
// falsePositiveEntryRepository.cloneEntries(clonedDossierTemplateId,dossierTemplateId);
}
}

View File

@ -11,9 +11,4 @@ public interface DossierAttributeConfigRepository extends JpaRepository<DossierA
List<DossierAttributeConfigEntity> findAllByDossierTemplateId(String dossierTemplateId);
void deleteByDossierTemplateId(String dossierTemplateId);
@Modifying
@Query("delete from DossierAttributeConfigEntity d where d.id in :dossierAttributeIds")
void deleteByDossierAtributeIds(List<String> dossierAttributeIds);
}

View File

@ -37,4 +37,11 @@ public interface EntryRepository extends JpaRepository<DictionaryEntryEntity, Lo
@Transactional
@Query(value = "update dictionary_entry set deleted = false, version = :version where type_id = :typeId and value in (:entries) returning value", nativeQuery = true)
List<String> undeleteEntries(String typeId, Set<String> entries, long version);
@Modifying(flushAutomatically = true, clearAutomatically = true)
@Transactional
@Query(value = "insert into dictionary_entry (value, version, deleted, type_id) " +
" select value, 1, false, :targetTypeId from dictionary_entry where type_id = :originTypeId and deleted = false", nativeQuery = true)
void cloneEntries(String targetTypeId, String originTypeId);
}

View File

@ -77,7 +77,9 @@ public class DictionaryController implements DictionaryResource {
if (removeCurrent) {
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, dictionaryEntryType);
long t1 = System.currentTimeMillis();
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
System.out.println("STUFF: {} "+(System.currentTimeMillis()-t1));
} else {
entryPersistenceService.addEntries(typeId, cleanEntries, currentVersion + 1, dictionaryEntryType);
}

View File

@ -0,0 +1,73 @@
package com.iqser.red.service.peristence.v1.server.integration.tests.performance;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.EntryRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.CloneDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntryType;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.ArrayList;
import java.util.List;
public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest {
@Autowired
private DictionaryClient dictionaryClient;
@Autowired
private DossierTemplateClient dossierTemplateClient;
@Autowired
private DossierTemplateTesterAndProvider dossierTemplateTesterAndProvider;
@Autowired
private TypeProvider typeProvider;
@Autowired
private EntryRepository entryRepository;
@Test
public void testAddToDictionary() {
var template = dossierTemplateTesterAndProvider.provideTestTemplate("test");
var type = typeProvider.testAndProvideType(template);
var fiveKEntries = generateEntries(5000);
var tenKEntries = generateEntries(10000);
long t1 = System.currentTimeMillis();
dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
long t2 = System.currentTimeMillis();
System.out.println("Time T1: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
t1 = System.currentTimeMillis();
dictionaryClient.addEntries(type.getTypeId(),tenKEntries , true, false, DictionaryEntryType.ENTRY);
t2 = System.currentTimeMillis();
System.out.println("Time T2: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
t1 = System.currentTimeMillis();
dictionaryClient.addEntries(type.getTypeId(),fiveKEntries , true, false, DictionaryEntryType.ENTRY);
t2 = System.currentTimeMillis();
System.out.println("Time T3: "+(t2-t1)+"ms counting: "+entryRepository.findByTypeIdAndVersionGreaterThan(type.getTypeId(),0).size()+" entries");
dossierTemplateClient.cloneDossierTemplate(template.getId(), new CloneDossierTemplateRequest());
}
private List<String> generateEntries(int count) {
List<String> entries = new ArrayList<>();
for (var i = 1; i <= count; i++) {
entries.add("Dictionary Entry: " + i);
}
return entries;
}
}

View File

@ -1,15 +1,23 @@
package com.iqser.red.service.peristence.v1.server.integration.utils;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.when;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
import com.iqser.red.service.peristence.v1.server.integration.client.ApplicationConfigClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.ApplicationConfig;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Before;
@ -26,29 +34,14 @@ import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.*;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
import com.iqser.red.service.peristence.v1.server.integration.client.FileClient;
import com.iqser.red.service.peristence.v1.server.utils.MetricsPrinterService;
import com.iqser.red.service.peristence.v1.server.utils.StorageIdUtils;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
@RunWith(SpringRunner.class)
@EnableFeignClients(basePackageClasses = FileClient.class)
@Import(AbstractPersistenceServerServiceTest.TestConfiguration.class)
@ -169,21 +162,21 @@ public abstract class AbstractPersistenceServerServiceTest {
}).when(pdfTronRedactionClient).processUntouchedDocument(any());
when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) ->{
when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) -> {
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream()));
});
when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((answer) ->{
when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((answer) -> {
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream()));
});
when(pdfTronRedactionClient.redactionPreviewDiff(Mockito.any())).thenAnswer((answer) ->{
when(pdfTronRedactionClient.redactionPreviewDiff(Mockito.any())).thenAnswer((answer) -> {
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
@ -255,8 +248,10 @@ public abstract class AbstractPersistenceServerServiceTest {
postgreSQLContainer.start();
var connectionStringDetails = "?serverTimezone=UTC&cachePrepStmts=true&useServerPrepStmts=true&rewriteBatchedStatements=true";
TestPropertyValues.of(
"spring.datasource.url=" + postgreSQLContainer.getJdbcUrl(),
"spring.datasource.url=" + postgreSQLContainer.getJdbcUrl() + connectionStringDetails,
"spring.datasource.username=" + postgreSQLContainer.getUsername(),
"spring.datasource.password=" + postgreSQLContainer.getPassword()
).applyTo(configurableApplicationContext.getEnvironment());

View File

@ -24,9 +24,10 @@ public class FileSystemBackArchiverTest {
SplittableRandom sr = new SplittableRandom();
var data = sr.doubles().limit(1024 * 1024 * 64).toArray();
for (int i = 0; i < 10; i++) {
log.info("At entry: {}, using {}MB of memory", i, (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024));
var data = sr.doubles().limit(1024 * 1024).toArray();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(bos);
oos.writeObject(data);