RED-10071 - Create system-managed entities automatically

- when exporting the types, do not export system managed types
- when importing a dossier template, ignore the system managed types from the archive
- when importing a dossier template as new or a new dossier template is created, also create the system managed types.
- unit tests updated
This commit is contained in:
corinaolariu 2024-10-10 11:16:33 +03:00
parent ec10bbdf8b
commit 0ed4759266
25 changed files with 753 additions and 187 deletions

View File

@ -273,7 +273,8 @@ public class DossierTemplateExportService {
// and 1 txt file for every type: entries, false positives and false recommendation
// remove the types related to dossiers and also the ones that are deleted
List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplate.getId(), false);
// also the ones that are system - managed
List<TypeEntity> dossierTypes = dictionaryPersistenceService.getAllTypesForDossierTemplateWithoutSystemManaged(dossierTemplate.getId(), false);
for (TypeEntity typeEntity : dossierTypes) {
// log.info("type: " + typeEntity.getType() + " " + typeEntity.getDossierId() + " " + typeEntity.isDeleted());
entityTypeExportService.addEntityTypeToArchive(fileSystemBackedArchiver, typeEntity, folder);

View File

@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.EntityTypeImportModel;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.FileAttributesGeneralConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.WatermarkEntity;
@ -65,6 +66,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.CreateOrUpdateDossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.DossierStatusInfo;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.model.DroolsValidation;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.databasetenantcommons.providers.utils.MagicConverter;
@ -102,6 +104,7 @@ public class DossierTemplateImportService {
ComponentMappingService componentMappingService;
ComponentDefinitionPersistenceService componentDefinitionPersistenceService;
EntityTypeImportService entityTypeImportService;
SystemManagedTypesImport systemManagedTypesImport;
public String importDossierTemplate(ImportDossierTemplateRequest request) {
@ -276,7 +279,9 @@ public class DossierTemplateImportService {
setComponentMappings(dossierTemplateId, request.getComponentMappings());
importStep = 12;
entityTypeImportService.updateTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
entityTypeImportService.updateTypes(dossierTemplateId,
null,
cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(), systemManagedTypesImport.getSystemManagedTypeList()));
} else {
@ -392,7 +397,15 @@ public class DossierTemplateImportService {
//set types
importStep = 12;
entityTypeImportService.importEntityTypes(dossierTemplateId, null, request.getEntityTypeImportModel());
// import system managed entity types
EntityTypeImportModel systemManagedTypesModel = new EntityTypeImportModel();
systemManagedTypesModel.getTypes().addAll(systemManagedTypesImport.getSystemManagedTypeList());
entityTypeImportService.importEntityTypes(dossierTemplateId, null, systemManagedTypesModel);
// import the rest of entity types
entityTypeImportService.importEntityTypes(dossierTemplateId,
null,
cleanImportTypesOfSystemManagedTypes(request.getEntityTypeImportModel(),
systemManagedTypesImport.getSystemManagedTypeList()));
}
@ -476,7 +489,7 @@ public class DossierTemplateImportService {
private void setDataFormats(ImportTemplateResult request, String dossierTemplateId) {
if(request.getDateFormats() != null) {
if (request.getDateFormats() != null) {
List<DateFormatPatternErrorMessage> dateFormatPatternErrorMessages = dateFormatsValidationService.validateDateFormats(request.getDateFormats());
if (!dateFormatPatternErrorMessages.isEmpty()) {
throw new BadRequestException("The date formats file contains errors");
@ -646,4 +659,30 @@ public class DossierTemplateImportService {
return dossierStatusPersistenceService.createOrUpdateDossierStatus(dossierStatusRequest);
}
private EntityTypeImportModel cleanImportTypesOfSystemManagedTypes(EntityTypeImportModel importModel, List<Type> systemManagedTypes) {
Set<String> systemManagedTypesByName = systemManagedTypes.stream()
.map(Type::getType)
.collect(Collectors.toSet());
Set<String> systemManagedTypesIdsFromImport = importModel.getTypes()
.stream()
.filter(t -> systemManagedTypesByName.contains(t.getType()) || t.isSystemManaged())
.map(Type::getType)
.collect(Collectors.toSet());
if (!systemManagedTypesIdsFromImport.isEmpty()) {
importModel.getTypes().removeIf(t -> systemManagedTypesIdsFromImport.contains(t.getType()));
importModel.getEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
importModel.getFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
importModel.getFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
importModel.getDeletedEntries().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
importModel.getDeletedFalsePositives().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
importModel.getDeletedFalseRecommendations().keySet().removeIf(key -> systemManagedTypesIdsFromImport.contains(key));
}
return importModel;
}
}

View File

@ -47,6 +47,7 @@ public class EntityTypeImportService {
List<TypeEntity> currentTypes = dossierTemplatePersistenceService.getTypesForDossierTemplate(dossierTemplateId)
.stream()
.filter(t -> t.getDossierId() == null)
.filter(t -> !t.isSystemManaged())
.collect(Collectors.toList());
Set<String> currentTypesId = currentTypes.stream()
@ -145,28 +146,10 @@ public class EntityTypeImportService {
private void deleteTypes(List<TypeEntity> currentTypes, Set<String> typeIdsAdded) {
Set<String> currentTypesIdSystemManaged = currentTypes.stream()
.filter(TypeEntity::isSystemManaged)
.map(TypeEntity::getId)
.collect(Collectors.toSet());
// for types system managed just delete the entries
Set<String> entriesToRemove = currentTypesIdSystemManaged.stream()
.filter(t -> !typeIdsAdded.contains(t))
.collect(Collectors.toSet());
entriesToRemove.forEach(typeId -> {
var currentVersion = dictionaryManagementService.getCurrentVersion(typeId);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.ENTRY);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_POSITIVE);
entryPersistenceService.deleteAllEntriesForTypeId(typeId, currentVersion + 1, DictionaryEntryType.FALSE_RECOMMENDATION);
dictionaryPersistenceService.incrementVersion(typeId);
typeIdsAdded.add(typeId); // added to the list, since the type can not be deleted
});
Set<String> typesToRemove = currentTypes.stream()
.filter(t -> !t.isDeleted()) // remove the ones already soft deleted
.map(TypeEntity::getId)
.filter(t -> !typeIdsAdded.contains(t)) // exclude the type ids already added from the import
.filter(t -> !currentTypesIdSystemManaged.contains(t)) // exclude the types system managed
.collect(Collectors.toSet());
typesToRemove.forEach(dictionaryManagementService::deleteType);
}

View File

@ -0,0 +1,59 @@
package com.iqser.red.service.persistence.management.v1.processor.dataexchange.service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class SystemManagedTypesImport {
@Value("${application.type}")
private String applicationType;
private final ObjectMapper objectMapper = ObjectMapperFactory.create();
private List<Type> rmSystemManagedTypeList = new ArrayList<>();
// List<Type> dmSystemManagedTypeList = new ArrayList<>();
@PostConstruct
public void init() {
Resource redactManagerResource = new ClassPathResource("files/systemtypes/rm/systemTypes.json");
// Resource docuMineResource = new ClassPathResource("files/systemtypes/dm/systemTypes.json");
try {
rmSystemManagedTypeList = objectMapper.readValue(redactManagerResource.getContentAsByteArray(), new TypeReference<>() {
});
// dmSystemManagedTypeList = objectMapper.readValue(docuMineResource.getContentAsByteArray(), new TypeReference<>() {
// });
} catch (IOException e) {
log.info("Could not read the system managed types for dossier templates error: {}", e.getMessage());
}
}
public List<Type> getSystemManagedTypeList() {
// if (applicationType.equals("DocuMine")) {
// return dmSystemManagedTypeList;
// }
return this.rmSystemManagedTypeList;
}
}

View File

@ -8,6 +8,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateExportService;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateImportService;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.SystemManagedTypesImport;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
@ -15,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.models.ExportDownloadRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -29,10 +31,25 @@ public class DossierTemplateManagementService {
private final DossierTemplatePersistenceService dossierTemplatePersistenceService;
private final DossierTemplateCloneService dossierTemplateCloneService;
private final SystemManagedTypesImport systemManagedTypesImport;
private final DictionaryManagementService dictionaryManagementService;
public DossierTemplate createOrUpdateDossierTemplate(CreateOrUpdateDossierTemplateRequest dossierTemplate) {
return convert(dossierTemplatePersistenceService.createOrUpdateDossierTemplate(dossierTemplate), DossierTemplate.class);
DossierTemplate dossierTemplate1 = convert(dossierTemplatePersistenceService.createOrUpdateDossierTemplate(dossierTemplate), DossierTemplate.class);
if (dossierTemplate.getDossierTemplateId() == null) {
// import system managed entity types to this dossier template
for (Type type : systemManagedTypesImport.getSystemManagedTypeList()) {
type.setDossierTemplateId(dossierTemplate1.getId());
type.setDossierId(null);
dictionaryManagementService.addType(type);
}
}
return dossierTemplate1;
}
@ -66,7 +83,6 @@ public class DossierTemplateManagementService {
}
public DossierTemplate importDossierTemplate(ImportDossierTemplateRequest request) {
String dossierTemplateId = dossierTemplateImportService.importDossierTemplate(request);

View File

@ -178,6 +178,13 @@ public class DictionaryPersistenceService {
}
public List<TypeEntity> getAllTypesForDossierTemplateWithoutSystemManaged(String dossierTemplateId, boolean includeDeleted) {
var types = typeRepository.getAllTypesForDossierTemplateWithoutSystemManagedTypes(dossierTemplateId);
return filterDeleted(types, includeDeleted);
}
public List<TypeEntity> getAllTypesForDossier(String dossierId, boolean includeDeleted) {
var types = typeRepository.findByDossierId(dossierId);
@ -319,13 +326,15 @@ public class DictionaryPersistenceService {
@Transactional
public void setExperimentalTrue(String typeId){
typeRepository.findById(typeId).ifPresent(type -> {
type.setVersion(type.getVersion() + 1);
type.setSystemManaged(true);
type.setExperimental(true);
typeRepository.saveAndFlush(type);
});
public void setExperimentalTrue(String typeId) {
typeRepository.findById(typeId)
.ifPresent(type -> {
type.setVersion(type.getVersion() + 1);
type.setSystemManaged(true);
type.setExperimental(true);
typeRepository.saveAndFlush(type);
});
}
}

View File

@ -44,6 +44,9 @@ public interface TypeRepository extends JpaRepository<TypeEntity, String> {
@Query("select t from TypeEntity t where t.dossierTemplateId = :dossierTemplateId and t.dossierId is null")
List<TypeEntity> getAllTypesForDossierTemplate(@Param("dossierTemplateId") String dossierTemplateId);
@Query("select t from TypeEntity t where t.dossierTemplateId = :dossierTemplateId and t.dossierId is null and t.systemManaged = false")
List<TypeEntity> getAllTypesForDossierTemplateWithoutSystemManagedTypes(@Param("dossierTemplateId") String dossierTemplateId);
@Query("select t from TypeEntity t where t.dossierTemplateId = :dossierTemplateId or t.dossierId = :dossierId")
List<TypeEntity> findAllTypesByDossierTemplateIdOrDossierId(@Param("dossierTemplateId") String dossierTemplateId, @Param("dossierId") String dossierId);

View File

@ -0,0 +1,207 @@
[
{
"id": "formula:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "formula",
"label": "Formula",
"hexColor": "#036ffc",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 1002,
"description": "Empty dictionary used to configure formula colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": true,
"hint": true,
"recommendation": false
},
{
"id": "graphic:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "graphic",
"label": "Graphic",
"hexColor": "#ffbdfd",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 998,
"description": "Empty dictionary used to configure graphic colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": true,
"caseInsensitive": true,
"recommendation": false,
"experimental": true
},
{
"id": "image:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "image",
"label": "Image",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 999,
"description": "Empty dictionary used to configure image colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": true,
"hint": true,
"recommendation": false
},
{
"id": "imported_redaction:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "imported_redaction",
"label": "Imported Redaction",
"hexColor": "#f0f0c0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 9999,
"description": "Redaction Annotations that were imported from documents",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": true,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": false,
"hint": false,
"recommendation": false
},
{
"id": "logo:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "logo",
"label": "Logo",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 1001,
"description": "Empty dictionary used to configure logo colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": true,
"hint": true,
"recommendation": false
},
{
"id": "manual:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "manual",
"label": "Manual Redactions",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 10000,
"description": "Manual Redactions",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": false,
"caseInsensitive": false,
"recommendation": false
},
{
"id": "ocr:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "ocr",
"label": "Ocr",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 1000,
"description": "Empty dictionary used to configure ocr colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": true,
"hint": true,
"recommendation": false
},
{
"id": "signature:4e4987e9-a285-40b8-9ae1-46464b37d322",
"type": "signature",
"label": "Signature",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#aaaaaa",
"rank": 1003,
"description": "Empty dictionary used to configure signature colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierDictionaryOnly": false,
"dossierTemplateId": "4e4987e9-a285-40b8-9ae1-46464b37d322",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"experimental": false,
"caseInsensitive": true,
"hint": true,
"recommendation": false
}
]

View File

@ -0,0 +1,195 @@
[
{
"id": "formula:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "formula",
"label": "Formula",
"hexColor": "#036ffc",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 1002,
"description": "Empty dictionary used to configure formula colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": true,
"caseInsensitive": true,
"recommendation": false
},
{
"id": "graphic:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "graphic",
"label": "Graphic",
"hexColor": "#ffbdfd",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 998,
"description": "Empty dictionary used to configure graphic colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": true,
"caseInsensitive": true,
"recommendation": false,
"experimental": true
},
{
"id": "image:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "image",
"label": "Image",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 999,
"description": "Empty dictionary used to configure image colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": true,
"caseInsensitive": true,
"recommendation": false
},
{
"id": "imported_redaction:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "imported_redaction",
"label": "Imported Redaction",
"hexColor": "#f0f0c0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 9999,
"description": "Redaction Annotations that were imported from documents",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": true,
"softDeletedTime": null,
"hint": false,
"caseInsensitive": false,
"recommendation": false
},
{
"id": "logo:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "logo",
"label": "Logo",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 1001,
"description": "Empty dictionary used to configure logo colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": false,
"caseInsensitive": true,
"recommendation": false
},
{
"id": "manual:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "manual",
"label": "Manual Redactions",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 10000,
"description": "Manual Redactions",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": false,
"caseInsensitive": false,
"recommendation": false
},
{
"id": "ocr:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "ocr",
"label": "Ocr",
"hexColor": "#bdd6ff",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 1000,
"description": "Empty dictionary used to configure ocr colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": true,
"caseInsensitive": true,
"recommendation": false
},
{
"id": "signature:50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"type": "signature",
"label": "Signature",
"hexColor": "#9398a0",
"recommendationHexColor": "#8df06c",
"skippedHexColor": "#c498fa",
"rank": 1003,
"description": "Empty dictionary used to configure signature colors.",
"version": 1,
"addToDictionaryAction": false,
"dossierTemplateId": "50f0087a-d999-45b2-b9c9-d9c62a6fb66c",
"dossierId": null,
"entries": [],
"falsePositiveEntries": [],
"falseRecommendationEntries": [],
"hasDictionary": false,
"systemManaged": true,
"autoHideSkipped": false,
"softDeletedTime": null,
"hint": false,
"caseInsensitive": true,
"recommendation": false
}
]

View File

@ -10,6 +10,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Assertions;
@ -408,10 +409,19 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.build());
assertThat(returnedtype1.isDossierDictionaryOnly()).isTrue();
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), null, false).getTypes().size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier.getId(), false).getTypes().size()).isEqualTo(2);
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), null, false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier.getId(), false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(2);
var dictionary = dictionaryClient.getDictionaryForType(returnedtype1.getType(), dossier.getDossierTemplateId(), dossier2.getId());
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier2.getId(), false).getTypes().size()).isEqualTo(2);
assertThat(dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier2.getId(), false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(2);
}
@ -451,7 +461,9 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
assertThat(actualEntries).usingComparator(new ListContentWithoutOrderAndWithoutDuplicatesComparator<>()).isEqualTo(entries);
dictionaryClient.deleteType(createdType.getType(), createdType.getDossierTemplateId());
assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(), null, false).getTypes().size()).isEqualTo(0);
assertThat(dictionaryClient.getAllTypes(createdType.getDossierTemplateId(), null, false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList()).size()).isEqualTo(0);
dictionaryClient.addType(type);
@ -1007,11 +1019,20 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.build());
assertThat(dtType.getRank()).isEqualTo(100);
assertThat(dtType.isDossierDictionaryOnly()).isFalse();
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(1);
dictionaryClient.deleteType(dtType.getType(), dtType.getDossierTemplateId());
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false).getTypes().size()).isEqualTo(0);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(0);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(1);
var dtType2 = dictionaryClient.addType(CreateTypeValue.builder()
.type("test2")
@ -1025,12 +1046,24 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.dossierTemplateId(dossierTemplate.getId())
.dossierDictionaryOnly(false)
.build());
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false).getTypes().size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().size()).isEqualTo(2);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(2);
assertThat(dtType2.getRank()).isEqualTo(dtType.getRank());
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes().size()).isEqualTo(2);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), true).getTypes().size()).isEqualTo(3);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(2);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), true)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(3);
}
@ -1063,7 +1096,7 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.dossierDictionaryOnly(false)
.build());
List<TypeValue> types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes();
List<TypeValue> types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList());
assertThat(types.size()).isEqualTo(1);
String dictionaryEntry = "entry1";
@ -1097,7 +1130,10 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
.build());
assertThat(dtType.getRank()).isEqualTo(100);
assertThat(dtType.isDossierDictionaryOnly()).isFalse();
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true).getTypes().size()).isEqualTo(1);
assertThat(dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true)
.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(1);
assertThatThrownBy(() -> dictionaryManagementService.addEntries(dtType.getTypeId(),
List.of("entry1", "entry2"),

View File

@ -23,12 +23,14 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.SystemManagedTypesImport;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.migration.RankDeDuplicationService;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.ComponentMappingService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateCloneService;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.DossierTemplateImportService;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
@ -107,6 +109,10 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
private DossierTemplateExportService dossierTemplateExportService;
private EntityTypeExportService entityTypeExportService;
private RankDeDuplicationService rankDeDuplicationService;
@MockBean
private DictionaryManagementService dictionaryManagementService;
@MockBean
private SystemManagedTypesImport systemManagedTypesImport;
ObjectMapper mapper;
@ -170,7 +176,9 @@ public class DossierTemplateCloneAndExportWithDuplicateRanksTest {
dossierTemplateManagementService = new DossierTemplateManagementService(dossierTemplateExportService,
dossierTemplateImportService,
dossierTemplatePersistenceService,
dossierTemplateCloneService);
dossierTemplateCloneService,
systemManagedTypesImport,
dictionaryManagementService);
rankDeDuplicationService = new RankDeDuplicationService(dossierTemplateManagementService, dictionaryPersistenceService);

View File

@ -14,6 +14,7 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@ -25,12 +26,15 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.mock.web.MockMultipartFile;
import com.iqser.red.service.peristence.v1.server.integration.client.DictionaryClient;
import com.iqser.red.service.peristence.v1.server.integration.client.DossierTemplateClient;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierTemplateManagementService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DossierTemplate;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.importexport.ImportDossierTemplateRequest;
import com.iqser.red.service.redaction.v1.model.DroolsBlacklistErrorMessage;
@ -51,6 +55,9 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
@Autowired
private DossierTemplateClient dossierTemplateClient;
@Autowired
private DictionaryClient dictionaryClient;
@SneakyThrows
@Test
@ -112,12 +119,34 @@ public class DossierTemplateImportTest extends AbstractPersistenceServerServiceT
@SneakyThrows
@Test
public void testDossierTemplateImport() {
public void testDossierTemplateImportAsNew() {
TenantContext.setTenantId("redaction");
var archive = loadFileFromClasspath("EFSA_sanitisation_GFL_v1.zip");
testDossierTemplateImport(archive);
// var archive = loadFileFromClasspath("EFSA_sanitisation_GFL_v1.zip");
var archive = loadFileFromClasspath("oldDossierTemplate.zip");
var request = ImportDossierTemplateRequest.builder().archive(archive).updateExistingDossierTemplate(false).userId("system").build();
DossierTemplate dossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
List<TypeValue> systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList());
assertThat(systemManagedTypes.size()).isEqualTo(8);
}
@SneakyThrows
@Test
public void testDossierTemplateImportAsUpdate() {
TenantContext.setTenantId("redaction");
// var archive = loadFileFromClasspath("EFSA_sanitisation_GFL_v1.zip");
var archive = loadFileFromClasspath("oldDossierTemplate.zip");
var request = ImportDossierTemplateRequest.builder().archive(archive).updateExistingDossierTemplate(true).userId("system").build();
DossierTemplate dossierTemplate = dossierTemplateManagementService.importDossierTemplate(request);
TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
List<TypeValue> systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList());
assertThat(systemManagedTypes.size()).isEqualTo(8);
}
@SneakyThrows

View File

@ -198,7 +198,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.get();
assertThat(dossierTemplateStats1.getDossierTemplateId()).isEqualTo(dossierTemplate2.getId());
assertThat(dossierTemplateStats1.getNumberOfDictionaries()).isEqualTo(2);
assertThat(dossierTemplateStats1.getNumberOfDictionaries()).isEqualTo(10);
List<DictionarySummary> dictionarySummaryList = dossierTemplateStats1.getDictionarySummaryList();
assertThat(dictionarySummaryList.stream()
.filter(d -> d.getType().equals(TYPE_ID_1))
@ -215,7 +215,7 @@ public class DossierTemplateStatsTest extends AbstractPersistenceServerServiceTe
.findAny()
.get();
assertThat(dossierTemplateStats3.getDossierTemplateId()).isEqualTo(dossierTemplate4.getId());
assertThat(dossierTemplateStats3.getNumberOfDictionaries()).isZero();
assertThat(dossierTemplateStats3.getNumberOfDictionaries()).isEqualTo(8);
// delete some entries, stats should be updated
var entries22 = new ArrayList<String>();

View File

@ -58,6 +58,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierAttr
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierStatusRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.DossierTemplateModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttributesConfig;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.WatermarkModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.CloneDossierTemplateRequest;
@ -147,6 +148,10 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
TypeResponse types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
List<TypeValue> systemManagedTypes = types.getTypes().stream().filter(TypeValue::isSystemManaged).collect(Collectors.toList());
assertThat(systemManagedTypes.size()).isEqualTo(8);
var allTemplates = dossierTemplateClient.getAllDossierTemplates();
assertThat(allTemplates.size()).isEqualTo(1);
assertThat(allTemplates.get(0)).isEqualTo(dossierTemplate);
@ -243,7 +248,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
.hexColor("#ddddd")
.recommendationHexColor("#cccccc")
.skippedHexColor("#cccccc")
.rank(999)
.rank(957)
.hint(false)
.recommendation(false)
.caseInsensitive(false)
@ -259,7 +264,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
.hexColor("#12345")
.recommendationHexColor("#6789a")
.skippedHexColor("#6789a")
.rank(1002)
.rank(1022)
.hint(false)
.recommendation(false)
.caseInsensitive(false)
@ -279,7 +284,9 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
dictionaryClient.getDictionaryForType(type2.getType(), type2.getDossierTemplateId(), dossier.getId());
var allTypes = dictionaryClient.getAllTypes(dossierTemplate.getId(), dossier.getId(), false).getTypes();
assertThat(allTypes.size()).isEqualTo(4);
assertThat(allTypes
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(4);
var typesWithRankOfType1 = allTypes.stream()
.filter(t -> t.getRank() == type.getRank())
.collect(Collectors.toList());
@ -441,7 +448,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
.hexColor("#ddddd")
.recommendationHexColor("#cccccc")
.skippedHexColor("#cccccc")
.rank(999)
.rank(957)
.hint(false)
.recommendation(false)
.caseInsensitive(false)
@ -456,7 +463,7 @@ public class DossierTemplateTest extends AbstractPersistenceServerServiceTest {
.hexColor("#12345")
.recommendationHexColor("#6789a")
.skippedHexColor("#6789a")
.rank(1002)
.rank(1022)
.hint(false)
.recommendation(false)
.caseInsensitive(false)

View File

@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Disabled;
@ -115,7 +116,9 @@ public class DossierTest extends AbstractPersistenceServerServiceTest {
var dossier = dossierTesterAndProvider.provideTestDossier();
var returnedTypes = dictionaryClient.getAllTypes(dossier.getDossierTemplateId(), dossier.getId(), false);
assertThat(returnedTypes.getTypes().size()).isEqualTo(0);
assertThat(returnedTypes.getTypes()
.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList())
.size()).isEqualTo(0);
var watermark = new WatermarkModel();
watermark.setName("watermark name");

View File

@ -1,6 +1,7 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@ -89,7 +90,7 @@ public class FileExchangeImportExportTest extends AbstractPersistenceServerServi
TenantContext.setTenantId(TENANT_1);
Path fileExchangeArchive = new ClassPathResource("files/fileexchange/EFSA sanitisation GFL v1_file-exchange.zip").getFile().toPath();
Path fileExchangeArchive = new ClassPathResource("files/fileexchange/fileExchange.zip").getFile().toPath();
String importedDossierTemplateId = fileExchangeImportService.importFileExchangeArchive(USER_ID, Files.readAllBytes(fileExchangeArchive)).getDossierTemplateId();
@ -130,15 +131,21 @@ public class FileExchangeImportExportTest extends AbstractPersistenceServerServi
@Test
@SneakyThrows
public void testDossierTemplateRoundtrip() {
public void testDossierTemplateRoundtripRM() {
Path dossierTemplate = new ClassPathResource("files/dossiertemplates/DossierTemplate.zip").getFile().toPath();
Path floraTemplate = new ClassPathResource("files/dossiertemplates/Flora.zip").getFile().toPath();
Path dossierTemplate = new ClassPathResource("files/dossiertemplates/dossierTemplateWithoutSystemTypes.zip").getFile().toPath();
testImportExport(dossierTemplate);
testImportExport(floraTemplate);
}
@Test
@SneakyThrows
public void testDossierTemplateRoundtripDM() {
Path floraTemplate = new ClassPathResource("files/dossiertemplates/FloraWithoutSystemTypes.zip").getFile().toPath();
testImportExport(floraTemplate);
}
@SneakyThrows
public void testImportExport(Path dossierTemplateExportArchive) {

View File

@ -406,7 +406,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");
typeProvider.testAndProvideType(dossierTemplate, null, "new-type", false, 2000);
var annotationId = "imagine_this_makes_sense";
var commentId = "commentId";
@ -543,7 +543,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
@ -589,7 +589,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier, fileName);
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2");
assertThat(fileClient.getDossierStatus(dossier.getId()).size()).isEqualTo(1);
@ -772,7 +772,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
var file = fileTesterAndProvider.testAndProvideFile(dossier);
String fileId = file.getId();
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual");
var type = typeProvider.testAndProvideType(dossierTemplate, null, "manual2"); // manual is a system managed type automatcally created
typeProvider.testAndProvideType(dossierTemplate, null, "new-type", false, 2000);
var annotationId = "imagine_this_makes_sense";
var annotation2Id = "imagine_this_makes_sense 2";
@ -1080,8 +1080,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
ClassPathResource newDocumentTextDataClassPath = new ClassPathResource("files/newDocumentData/new.DOCUMENT_TEXT.proto");
storageService.storeObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextDataClassPath.getInputStream());
DocumentStructureProto.DocumentStructure documentStructure = storageService.readProtoObject(TenantContext.getTenantId(),
newDocumentStructureStorageId,
DocumentStructureProto.DocumentStructure.parser());
@ -1098,9 +1096,9 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
newDocumentPositionDataStorageId,
DocumentPositionDataProto.AllDocumentPositionData.parser());
}
@Test
@SneakyThrows
@Disabled
@ -1174,6 +1172,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
@Test
@Disabled
public void testParseProtoFiles() throws Exception {
String tempDir = "/tmp";
Path documentStructurePath = Path.of(tempDir, "new.DOCUMENT_STRUCTURE.proto");
@ -1186,14 +1185,10 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
byte[] documentTextBytes = Files.readAllBytes(documentTextPath);
byte[] documentPositionBytes = Files.readAllBytes(documentPositionPath);
DocumentStructureProto.DocumentStructure documentStructure =
DocumentStructureProto.DocumentStructure.parseFrom(documentStructureBytes);
DocumentPageProto.AllDocumentPages allDocumentPages =
DocumentPageProto.AllDocumentPages.parseFrom(documentPagesBytes);
DocumentTextDataProto.AllDocumentTextData allDocumentTextData =
DocumentTextDataProto.AllDocumentTextData.parseFrom(documentTextBytes);
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData =
DocumentPositionDataProto.AllDocumentPositionData.parseFrom(documentPositionBytes);
DocumentStructureProto.DocumentStructure documentStructure = DocumentStructureProto.DocumentStructure.parseFrom(documentStructureBytes);
DocumentPageProto.AllDocumentPages allDocumentPages = DocumentPageProto.AllDocumentPages.parseFrom(documentPagesBytes);
DocumentTextDataProto.AllDocumentTextData allDocumentTextData = DocumentTextDataProto.AllDocumentTextData.parseFrom(documentTextBytes);
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = DocumentPositionDataProto.AllDocumentPositionData.parseFrom(documentPositionBytes);
}
}

View File

@ -2597,12 +2597,12 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var file = fileTesterAndProvider.testAndProvideFile(dossier);
var typeImage = typeProvider.testAndProvideType(dossierTemplate, null, "image", false, 999);
var typeLogo = typeProvider.testAndProvideType(dossierTemplate, null, "logo", false, 1001);
var typeImage = "image";
var typeLogo = "logo";
var entityLogEntry = EntityLogEntry.builder()
.id("annotationId")
.type(typeImage.getType())
.type(typeImage)
.entryType(EntryType.IMAGE_HINT)
.state(EntryState.SKIPPED)
.value("Image:Other")
@ -2647,7 +2647,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
.annotationId("annotationId")
.legalBasis("")
.section("[31]: Section: Images: Regular Images: Formular")
.type(typeLogo.getType())
.type(typeLogo)
.value("Image:Other")
.build();
@ -2657,7 +2657,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertTrue(allManualRedactions.getRecategorizations()
.stream()
.anyMatch(manualRecategorization -> manualRecategorization.getType().equals(typeLogo.getType())));
.anyMatch(manualRecategorization -> manualRecategorization.getType().equals(typeLogo)));
}
@ -3341,115 +3341,75 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
String newLegal = "new Legal";
String otherSection = "other section";
String comment1 = "Recategorizing Luke Skywalker37";
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value("Luke Skywalker37")
.comment(comment1)
.build(),
false);
assertEquals(1, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection());
Long commentId1 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId1);
Optional<CommentEntity> commentEntity1 = commentRepository.findById(commentId1);
assertTrue(commentEntity1.isPresent());
assertEquals(comment1, commentEntity1.get().getText());
ManualRedactionResponse manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value("Luke Skywalker37")
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 1);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getSection(), otherSection);
String comment2 = "Recategorizing Darth Vader with Legal 3";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originLegalBases(Set.of(legal3))
.comment(comment2)
.build(),
false);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection());
Long commentId2 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId2);
Optional<CommentEntity> commentEntity2 = commentRepository.findById(commentId2);
assertTrue(commentEntity2.isPresent());
assertEquals(comment2, commentEntity2.get().getText());
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originLegalBases(Set.of(legal3))
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(1).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(29).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(79).getEntityLogEntry().getSection(), otherSection);
String comment3 = "Recategorizing Darth Vader with type2";
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(
dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originTypes(Set.of(type2.getType()))
.comment(comment3)
.build(),
false);
assertEquals(101, manualRedactionResponse.getManualAnnotationResponses().size());
assertEquals(newType.getType(), manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType());
assertEquals(newLegal, manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis());
assertEquals(otherSection, manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection());
Long commentId3 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId3);
Optional<CommentEntity> commentEntity3 = commentRepository.findById(commentId3);
assertTrue(commentEntity3.isPresent());
assertEquals(comment3, commentEntity3.get().getText());
manualRedactionResponse = manualRedactionClient.recategorizeBulkLocal(dossier.getId(),
file.getId(),
RecategorizationBulkLocalRequestModel.builder()
.rectangle(false)
.type(newType.getType())
.legalBasis(newLegal)
.section(otherSection)
.value(darthVader)
.originTypes(Set.of(type2.getType()))
.build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 101);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(0).getEntityLogEntry().getType(), newType.getType());
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(36).getEntityLogEntry().getLegalBasis(), newLegal);
assertEquals(manualRedactionResponse.getManualAnnotationResponses()
.get(98).getEntityLogEntry().getSection(), otherSection);
String comment4 = "Removing all Darth Vader annotations";
manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(
dossier.getId(),
file.getId(),
RemoveRedactionBulkLocalRequestModel.builder()
.rectangle(false)
.value(darthVader)
.comment(comment4)
.build(),
false);
assertEquals(202, manualRedactionResponse.getManualAnnotationResponses().size());
Long commentId4 = manualRedactionResponse.getManualAnnotationResponses()
.get(0).getCommentId();
assertNotNull(commentId4);
Optional<CommentEntity> commentEntity4 = commentRepository.findById(commentId4);
assertTrue(commentEntity4.isPresent());
assertEquals(comment4, commentEntity4.get().getText());
manualRedactionResponse = manualRedactionClient.removeRedactionBulkLocal(dossier.getId(),
file.getId(),
RemoveRedactionBulkLocalRequestModel.builder().rectangle(false).value(darthVader).build(),
false);
assertEquals(manualRedactionResponse.getManualAnnotationResponses().size(), 202);
}
private void testRectangleRedactionsBulkLocal(DossierTemplateModel dossierTemplate, Dossier dossier, FileStatus file) {
whenGetEntityLogInvocation();
var typeManual = typeProvider.testAndProvideType(dossierTemplate, dossier, "manual", false, 70);
var typeManual = "manual";
var newType = typeProvider.testAndProvideType(dossierTemplate, dossier, "newType", false, 71);
String value = "non-readable content";
@ -3467,7 +3427,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
EntityLogEntry entityLogEntry = EntityLogEntry.builder()
.id("RectangleAnnotation" + page)
.type(typeManual.getType())
.type(typeManual)
.value(value)
.entryType(EntryType.AREA)
.state(EntryState.APPLIED)
@ -3497,7 +3457,7 @@ public class ManualRedactionTest extends AbstractPersistenceServerServiceTest {
assertNotNull(manualRedactionResponse);
assertEquals(3, manualRedactionResponse.getManualAnnotationResponses().size());
for (ManualAnnotationResponse response : manualRedactionResponse.getManualAnnotationResponses()) {
assertEquals(typeManual.getType(), response.getEntityLogEntry().getType());
assertEquals(typeManual, response.getEntityLogEntry().getType());
assertEquals("new Legal Basis", response.getEntityLogEntry().getLegalBasis());
assertEquals("new Section", response.getEntityLogEntry().getSection());
}

View File

@ -26,6 +26,7 @@ import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPers
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.service.DossierService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeResponse;
import com.iqser.red.service.persistence.service.v1.api.shared.model.UpdateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@ -83,7 +84,7 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
dictionaryClient.deleteType(type.getType(), type.getDossierTemplateId());
var typesForTemplate = dictionaryClient.getAllTypes(loadedType.getDossierTemplateId(), null, false).getTypes();
assertThat(typesForTemplate).isEmpty();
assertThat(typesForTemplate).hasSize(8);
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate);
var dossierType = typeProvider.testAndProvideType(dossierTemplate, dossier, "dossier-dict");
@ -143,14 +144,14 @@ public class TypeTest extends AbstractPersistenceServerServiceTest {
var type2 = typeProvider.testAndProvideType(dossierTemplate, "type2", 101);
var types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
assertEquals(types.getTypes().size(), 2);
assertEquals(types.getTypes().size(), 10);
assertTrue(types.getTypes()
.stream()
.allMatch(typeValue -> typeValue.getSoftDeletedTime() == null));
dictionaryClient.deleteType(type2.getType(), dossierTemplate.getId());
types = dictionaryClient.getAllTypes(dossierTemplate.getId(), null, true);
assertEquals(types.getTypes().size(), 2);
assertEquals(types.getTypes().size(), 10);
assertNull(types.getTypes()
.stream()
.filter(t -> t.getType().equals(type1.getType()))

View File

@ -80,9 +80,11 @@ public class EntityPerformanceTest extends AbstractPersistenceServerServiceTest
log.info("Clone Time: {}", (t2 - t1));
var types = dictionaryClient.getAllTypes(cloned.getId(), null, false).getTypes();
var typesWithoutSystemManaged = types.stream().filter(t -> !t.isSystemManaged()).collect(Collectors.toList());
assertThat(types.size()).isEqualTo(1);
for (var clonedType : types) {
assertThat(typesWithoutSystemManaged.size()).isEqualTo(1);
assertThat(types.size()).isEqualTo(9); // includes the system managed types
for (var clonedType : typesWithoutSystemManaged) {
var found = entryRepository.findByTypeIdAndVersionGreaterThan(clonedType.getTypeId(), 0);
assertThat(found.size()).isEqualTo(5000);
}

View File

@ -66,9 +66,11 @@ import com.iqser.red.service.persistence.management.v1.processor.client.redactio
import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.RedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.client.searchservice.SearchClient;
import com.iqser.red.service.persistence.management.v1.processor.client.tenantusermanagementservice.UsersClient;
import com.iqser.red.service.persistence.management.v1.processor.dataexchange.service.SystemManagedTypesImport;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ApplicationConfigurationEntity;
import com.iqser.red.service.persistence.management.v1.processor.roles.ApplicationRoles;
import com.iqser.red.service.persistence.management.v1.processor.service.ApplicationConfigService;
import com.iqser.red.service.persistence.management.v1.processor.service.DictionaryManagementService;
import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.ApplicationConfigRepository;
@ -273,6 +275,10 @@ public abstract class AbstractPersistenceServerServiceTest {
protected ResizeRedactionRepository resizeRedactionRepository;
@MockBean
protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
@Autowired
private SystemManagedTypesImport systemManagedTypesImport;
@Autowired
private DictionaryManagementService dictionaryManagementService;
private static String[] getAllRoles() {