Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

Merge branch 'RED-6734' into 'master'

Browse Source 
RED-6734 - Get merged dossier and template dictionaries

Closes RED-6734

See merge request redactmanager/persistence-service!11
This commit is contained in:
Corina Olariu 2023-06-20 11:08:14 +02:00
commit e5d19b98d7
12 changed files with 175 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
persistence-service-v1/persistence-service-external-api-impl-v1/src/main/java/com/iqser/red/persistence/service/v1/external/api/impl/controller/DictionaryController.java vendored
View File

@ -1,4 +1,4 @@
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
package com.iqser.red.persistence.service.v1.external.api.impl.controller;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@ -291,6 +291,13 @@ public class DictionaryController implements DictionaryResource {
return dictionaryService.getDictionaryForType(type, dossierTemplateId, dossierId);
}
@Override
public List<Dictionary> getMergedDictionaries(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
@RequestParam(value = DOSSIER_ID_PARAMETER_NAME) String dossierId) {
return dictionaryService.getMergedDictionaryForType(dossierTemplateId, dossierId);
}
@Override
public void setColors(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId, @RequestBody Colors colors) {

6
persistence-service-v1/persistence-service-external-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/external/resource/DictionaryResource.java vendored
View File

@ -47,6 +47,7 @@ public interface DictionaryResource {
String UPLOAD = "/upload";
String DOWNLOAD = "/download";
String MERGED = "/merged";
String DELETE = "/delete";
String COLOR_REST_PATH = ExternalApi.BASE_PATH + "/color";
@ -172,6 +173,11 @@ public interface DictionaryResource {
@RequestParam(value = DOSSIER_ID_PARAMETER_NAME, required = false) String dossierId,
@RequestParam(value = DICTIONARY_ENTRY_TYPE_PARAM, required = false, defaultValue = DEFAULT_DICTIONARY_ENTRY_TYPE) DictionaryEntryType dictionaryEntryType);
@GetMapping(value = DICTIONARY_REST_PATH + MERGED + DOSSIER_TEMPLATE_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Retrieves and merge all dictionary entries for the given dossier template and dossier", description = "None")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "Successfully retrieved all the dictionary entries"), @ApiResponse(responseCode = "400", description = "Request contains error."), @ApiResponse(responseCode = "404", description = "The entry type is not found.")})
List<Dictionary> getMergedDictionaries(@PathVariable(DOSSIER_TEMPLATE_PARAMETER_NAME) String dossierTemplateId,
@RequestParam(value = DOSSIER_ID_PARAMETER_NAME) String dossierId);
@ResponseStatus(HttpStatus.NO_CONTENT)
@Operation(summary = "Set system colors for redaction")

2
persistence-service-v1/persistence-service-internal-api-impl-v1/src/main/java/com/iqser/red/service/persistence/v1/internal/api/controller/DictionaryInternalController.java
View File

@ -8,12 +8,12 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.service.ColorsService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.internal.resources.DictionaryResource;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;

2
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryManagementService.java
View File

@ -17,6 +17,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.BaseDictionaryEntry;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
@ -28,7 +29,6 @@ import com.iqser.red.service.persistence.management.v1.processor.utils.MagicConv
import com.iqser.red.service.persistence.management.v1.processor.utils.TextNormalizationUtilities;
import com.iqser.red.service.persistence.management.v1.processor.utils.TypeMapper;
import com.iqser.red.service.persistence.management.v1.processor.validation.DictionaryValidator;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;

97
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/DictionaryService.java
View File

@ -13,14 +13,23 @@ import static com.iqser.red.keycloak.commons.roles.ActionRoles.READ_DICTIONARY_T
import static com.iqser.red.keycloak.commons.roles.ActionRoles.WRITE_COLORS;
import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.ColorsEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.EntryPersistenceService;
@ -32,7 +41,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeRespons
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.UpdateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
@ -50,6 +58,7 @@ public class DictionaryService {
private final DictionaryPersistenceService dictionaryPersistenceService;
private final AccessControlService accessControlService;
private final EntryPersistenceService entryPersistenceService;
private final DictionaryMergeService dictionaryMergeService;
@PreAuthorize("hasAuthority('" + ADD_DICTIONARY_ENTRY + "')")
@ -165,14 +174,6 @@ public class DictionaryService {
}
@PreAuthorize("hasAuthority('" + ADD_UPDATE_DOSSIER_DICTIONARY_TYPE + "')")
public Type addDossierType(CreateTypeValue typeValue, String dossierId) {
accessControlService.verifyUserIsMemberOrApprover(dossierId);
return addType(typeValue, dossierId);
}
@PreAuthorize("hasAuthority('" + DELETE_DICTIONARY_TYPE + "')")
public void deleteGlobalType(String type, String dossierTemplateId) {
@ -289,6 +290,84 @@ public class DictionaryService {
}
}
@PreAuthorize("hasAuthority('" + READ_DICTIONARY_TYPES + "')")
public List<Dictionary> getMergedDictionaryForType(String dossierTemplateId, String dossierId) {
try {
if (dossierId != null) {
accessControlService.verifyUserHasViewPermissions(dossierId);
}
var dossierTemplateDictionaries = dictionaryPersistenceService.getAllTypesForDossierTemplate(dossierTemplateId, false);
var dossierDictionaries = dictionaryPersistenceService.getAllTypesForDossier(dossierId, false);
var mergedDictionaries = dictionaryMergeService.getMergedDictionary(convertTypes(dossierTemplateDictionaries), convertTypes(dossierDictionaries));
return convertMergedDictionaries(mergedDictionaries, dossierTemplateId, dossierId);
} catch (AccessDeniedException e) {
throw new NotFoundException("Object not found");
}
}
private List<Dictionary> convertMergedDictionaries (List<CommonsDictionaryModel> commonsDictionaries, String dossierTemplateId, String dossierId) {
List<Dictionary> dictionaries = new ArrayList<>();
commonsDictionaries.forEach(cdm -> {
var typeId = toTypeId(cdm.getType(), dossierTemplateId, dossierId);
var entity = dictionaryPersistenceService.getType(typeId);
Dictionary dict = Dictionary.builder()
.entries(cdm.getEntries()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()))
.falsePositiveEntries(cdm.getFalsePositives()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()))
.falseRecommendationEntries(cdm.getFalseRecommendations()
.stream()
.map(DictionaryEntry::getValue)
.collect(Collectors.toList()))
.hexColor(entity.getHexColor())
.recommendationHexColor(entity.getRecommendationHexColor())
.skippedHexColor(entity.getSkippedHexColor())
.dossierTemplateId(dossierTemplateId)
.rank(cdm.getRank())
.hint(cdm.isHint())
.caseInsensitive(cdm.isCaseInsensitive())
.recommendation(entity.isRecommendation())
.description(entity.getDescription())
.addToDictionaryAction(entity.isAddToDictionaryAction())
.label(entity.getLabel())
.hasDictionary(entity.isHasDictionary())
.systemManaged(entity.isSystemManaged())
.autoHideSkipped(entity.isAutoHideSkipped())
.dossierDictionaryOnly(entity.isDossierDictionaryOnly())
.build();
dictionaries.add(dict);
});
return dictionaries;
}
private List<CommonsDictionaryModel> convertTypes(List<TypeEntity> dictionaries) {
return dictionaries.stream()
.map(dm -> CommonsDictionaryModel.builder()
.type(dm.getType())
.rank(dm.getRank())
.color(ColorUtils.convertColor(dm.getHexColor()))
.caseInsensitive(dm.isCaseInsensitive())
.hint(dm.isHint())
.entries(getCommonsEntries(dm.getId(), DictionaryEntryType.ENTRY))
.falsePositives(getCommonsEntries(dm.getId(), DictionaryEntryType.FALSE_POSITIVE))
.falseRecommendations(getCommonsEntries(dm.getId(), DictionaryEntryType.FALSE_RECOMMENDATION))
.build()).collect(Collectors.toList());
}
private Set<DictionaryEntryModel> getCommonsEntries(String typeId, DictionaryEntryType entryType) {
var entries = MagicConverter.convert(entryPersistenceService.getEntries(typeId, entryType, null), DictionaryEntry.class);
return CollectionUtils.isNotEmpty(entries) ? new HashSet<>(entries.stream()
.map(DictionaryEntryModel::new)
.collect(Collectors.toSet())) : new HashSet<>();
}
@PreAuthorize("hasAuthority('" + WRITE_COLORS + "')")
public void setColors(String dossierTemplateId, Colors colors) {

20
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/redactionlog/RedactionLogMergeService.java
View File

@ -15,6 +15,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.utils.ColorUtils;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@ -479,10 +480,10 @@ public class RedactionLogMergeService {
private float[] getColor(String type, Colors colors, boolean requested, boolean isRedaction, boolean skipped, List<Type> types) {
if (requested) {
return convertColor(colors.getRequestRemoveColor());
return ColorUtils.convertColor(colors.getRequestRemoveColor());
}
if (skipped || !isRedaction && !isHint(types, type)) {
return convertColor(colors.getSkippedColor());
return ColorUtils.convertColor(colors.getSkippedColor());
}
return getColor(types, type);
}
@ -491,9 +492,9 @@ public class RedactionLogMergeService {
private float[] getColorForManualAdd(String type, Colors colors, AnnotationStatus status, List<Type> types) {
if (status.equals(AnnotationStatus.REQUESTED)) {
return convertColor(colors.getRequestAddColor());
return ColorUtils.convertColor(colors.getRequestAddColor());
} else if (status.equals(AnnotationStatus.DECLINED)) {
return convertColor(colors.getSkippedColor());
return ColorUtils.convertColor(colors.getSkippedColor());
}
return getColor(types, type);
}
@ -504,10 +505,10 @@ public class RedactionLogMergeService {
var matchingTypes = getMatchingTypes(types, type);
Optional<Type> foundAndNotDeletedType = matchingTypes.stream().filter(t -> !isDeletedType(t)).findFirst();
if (foundAndNotDeletedType.isPresent()) {
return convertColor(foundAndNotDeletedType.get().getHexColor());
return ColorUtils.convertColor(foundAndNotDeletedType.get().getHexColor());
}
Optional<Type> firstDeletedType = matchingTypes.stream().findFirst();
return firstDeletedType.map(value -> convertColor(value.getHexColor())).orElseGet(() -> convertColor(DELETED_TYPE_COLOR));
return firstDeletedType.map(value -> ColorUtils.convertColor(value.getHexColor())).orElseGet(() -> ColorUtils.convertColor(DELETED_TYPE_COLOR));
}
@ -523,13 +524,6 @@ public class RedactionLogMergeService {
}
private float[] convertColor(String hex) {
Color color = Color.decode(hex);
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
}
private List<Type> getMatchingTypes(List<Type> types, String type) {
return types.stream().filter(t -> t.getType().equals(type)).collect(Collectors.toList());

7
persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/utils/ColorUtils.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.persistence.management.v1.processor.utils;
import java.awt.Color;
import java.util.Locale;
import java.util.regex.Pattern;
@ -23,4 +24,10 @@ public final class ColorUtils {
}
}
public static float[] convertColor(String hex) {
Color color = Color.decode(hex);
return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f};
}
}

7
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/Application.java
View File

@ -25,6 +25,7 @@ import com.iqser.red.keycloak.commons.DefaultKeyCloakCommonsConfiguration;
import com.iqser.red.keycloak.commons.KeyCloakSettings;
import com.iqser.red.persistence.service.v1.external.api.impl.PersistenceServiceExternalApiConfiguration;
import com.iqser.red.persistence.service.v1.external.api.impl.swagger.SwaggerAutoConfiguration;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.management.v1.processor.PersistenceServiceProcessorConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.cache.PersistenceServiceExternalApiCacheConfiguration;
import com.iqser.red.service.persistence.management.v1.processor.configuration.CleanupDownloadSchedulerConfiguration;
@ -83,4 +84,10 @@ public class Application {
};
}
@Bean
public DictionaryMergeService dictionaryMergeService() {
return new DictionaryMergeService();
}
}

43
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/DictionaryTest.java
View File

@ -20,6 +20,7 @@ import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvid
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.utils.multitenancy.TenantContext;
import com.iqser.red.service.persistence.service.v1.api.shared.model.CreateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.Dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.TypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.UpdateTypeValue;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@ -368,6 +369,48 @@ public class DictionaryTest extends AbstractPersistenceServerServiceTest {
assertThat(dictionary.getEntries().size()).isEqualTo(5);
}
@Test
public void testGetMergedDictionaries() {
var dossierTemplate = dossierTemplateTesterAndProvider.provideTestTemplate();
var dossier = dossierTesterAndProvider.provideTestDossier(dossierTemplate, "Dossier");
var type = typeProvider.testAndProvideType(dossierTemplate);
assertThat(type.getRecommendationHexColor()).isEqualTo("#aaaaaa");
assertThat(type.getSkippedHexColor()).isEqualTo("#aaaaaa");
assertThat(type.isDossierDictionaryOnly()).isTrue();
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("word1", "word2", "word3"), false, null, DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("false_positive1", "false_positive"), false, null, DictionaryEntryType.FALSE_POSITIVE);
dictionaryClient.addEntry(type.getType(),
type.getDossierTemplateId(),
List.of("false_recommendation1", "false_recommendation2"),
false,
null,
DictionaryEntryType.FALSE_RECOMMENDATION);
var loadedType1 = dictionaryClient.getDictionaryForType(type.getType(), type.getDossierTemplateId(), null);
assertThat(loadedType1.getEntries()).hasSize(3);
assertThat(loadedType1.getFalsePositiveEntries()).hasSize(2);
assertThat(loadedType1.getFalseRecommendationEntries()).hasSize(2);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("word1", "word2", "word4"), false, dossier.getDossierId(), DictionaryEntryType.ENTRY);
dictionaryClient.addEntry(type.getType(), type.getDossierTemplateId(), List.of("false_positive1", "false_positive3"), false, dossier.getDossierId(), DictionaryEntryType.FALSE_POSITIVE);
dictionaryClient.addEntry(type.getType(),
type.getDossierTemplateId(),
List.of("false_recommendation3", "false_recommendation4"),
false,
dossier.getDossierId(),
DictionaryEntryType.FALSE_RECOMMENDATION);
List<Dictionary> result = dictionaryClient.getMergedDictionaries(dossierTemplate.getDossierTemplateId(), dossier.getDossierId());
assertThat(result).isNotEmpty();
assertThat(result.size()).isEqualTo(1);
Dictionary dict = result.get(0);
assertThat(dict.getEntries().size()).isEqualTo(4);
assertThat(dict.getFalsePositiveEntries().size()).isEqualTo(3);
assertThat(dict.getEntries().size()).isEqualTo(4);
}
@Test
public void testCreateAndDeleteLargeDictionary() {

6
persistence-service-v1/persistence-service-shared-api-v1/pom.xml
View File

@ -73,6 +73,12 @@
<artifactId>jackson-commons</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>dictionary-merge-commons</artifactId>
<version>1.3.0</version>
</dependency>
<!-- test -->
<dependency>
<groupId>com.iqser.red.commons</groupId>

22
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/type/DictionaryEntry.java
View File

@ -1,22 +0,0 @@
package com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DictionaryEntry implements Serializable {
private long entryId;
private String value;
private long version;
private boolean deleted;
private String typeId;
}

1
persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/dossiertemplate/type/Type.java
View File

@ -5,6 +5,7 @@ import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import lombok.AllArgsConstructor;
import lombok.Builder;