Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6734 - Get merged dossier and template dictionaries

Browse Source 
- update to the removing of logic of merge dictionaries to dictionary-merge commons
This commit is contained in:
Corina Olariu 2023-06-20 15:27:11 +03:00
parent 57ba08d5b4
commit 778286e3b8
12 changed files with 61 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -12,7 +12,7 @@
<artifactId>redaction-service-api-v1</artifactId>
<properties>
<persistence-service.version>2.70.0</persistence-service.version>
<persistence-service.version>2.75.0</persistence-service.version>
</properties>
<dependencies>

6
redaction-service-v1/redaction-service-server-v1/pom.xml
View File

@ -27,6 +27,12 @@
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>dictionary-merge-commons</artifactId>
<version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.iqser.red.commons</groupId>
<artifactId>storage-commons</artifactId>

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java
View File

@ -9,6 +9,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig;
import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration;
@ -38,4 +39,10 @@ public class Application {
return new TimedAspect(registry);
}
@Bean
public DictionaryMergeService dictionaryMergeService() {
return new DictionaryMergeService();
}
}

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java
View File

@ -48,9 +48,11 @@ public class RedactionMessageReceiver {
// This prevents from endless retries oom errors.
if (message.getMessageProperties().isRedelivered()) {
throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
analyzeRequest.getDossierId(),
analyzeRequest.getFileId()));
analyzeRequest.getFileId());
fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorMessage, priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
throw new AmqpRejectAndDontRequeueException(errorMessage);
}
receiveAnalyzeRequest(analyzeRequest, priority);

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java
View File

@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.HashSet;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import lombok.AllArgsConstructor;
import lombok.Builder;

35
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntryModel.java
View File

@ -1,35 +0,0 @@
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import lombok.EqualsAndHashCode;
@EqualsAndHashCode(callSuper = false)
public class DictionaryEntryModel extends DictionaryEntry {
public DictionaryEntryModel() {
super();
}
public DictionaryEntryModel(long entryId, String value, long version, boolean deleted, String typeId) {
super(entryId, value, version, deleted, typeId);
}
public DictionaryEntryModel(DictionaryEntry entry) {
super(entry.getEntryId(), entry.getValue(), entry.getVersion(), entry.isDeleted(), entry.getTypeId());
}
@Override
@EqualsAndHashCode.Include
public String getValue() {
return super.getValue();
}
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java
View File

@ -5,7 +5,8 @@ import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import lombok.AllArgsConstructor;
import lombok.Data;

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,12 +1,11 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.awt.Color;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@ -19,13 +18,15 @@ import org.springframework.stereotype.Service;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
@ -51,6 +52,8 @@ public class DictionaryService {
private final RedactionServiceSettings settings;
private final DictionaryMergeService dictionaryMergeService;
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
@ -291,33 +294,24 @@ public class DictionaryService {
@Timed("redactmanager_getDeepCopyDictionary")
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
Map<String, DictionaryModel> mergedDictionaries = new HashMap<>();
List<DictionaryModel> mergedDictionaries;
var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
dossierTemplateRepresentation.getDictionary().forEach(dm -> mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm)));
var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
// merge dictionaries if they have same names
long dossierDictionaryVersion = -1;
if (dossierDictionaryExists(dossierId)) {
var dossierRepresentation = getDossierDictionary(dossierId);
dossierRepresentation.getDictionary().forEach(dm -> {
if (mergedDictionaries.containsKey(dm.getType())) {
//merge dictionary entries, false positives and false recommendations
Set<DictionaryEntryModel> dossierEntries = dm.getEntries();
mergedDictionaries.get(dm.getType()).getEntries().removeAll(dossierEntries);
mergedDictionaries.get(dm.getType()).getEntries().addAll(dossierEntries);
mergedDictionaries.get(dm.getType()).getFalsePositives().removeAll(dm.getFalsePositives());
mergedDictionaries.get(dm.getType()).getFalsePositives().addAll(dm.getFalsePositives());
mergedDictionaries.get(dm.getType()).getFalseRecommendations().removeAll(dm.getFalseRecommendations());
mergedDictionaries.get(dm.getType()).getFalseRecommendations().addAll(dm.getFalseRecommendations());
} else {
mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm));
}
});
var dossierDictionaries = dossierRepresentation.getDictionary();
mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), convertDictionaryModel(dossierDictionaries)));
dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
} else {
mergedDictionaries = new ArrayList<>();
dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
}
return new Dictionary(mergedDictionaries.values().stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()),
return new Dictionary(mergedDictionaries.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()),
DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
}
@ -373,4 +367,24 @@ public class DictionaryService {
}
}
private List<CommonsDictionaryModel> convertDictionaryModel(List<DictionaryModel> dictionaries) {
return dictionaries.stream().map(d -> CommonsDictionaryModel.builder()
.type(d.getType())
.rank(d.getRank())
.color(d.getColor())
.caseInsensitive(d.isCaseInsensitive())
.hint(d.isHint())
.isDossierDictionary(d.isDossierDictionary())
.entries(d.getEntries())
.falsePositives(d.getFalsePositives())
.falseRecommendations(d.getFalseRecommendations())
.build()).collect(Collectors.toList());
}
private List<DictionaryModel> convertCommonsDictionaryModel(List<CommonsDictionaryModel> commonsDictionaries) {
return commonsDictionaries.stream().map(cd ->
new DictionaryModel(cd.getType(), cd.getRank(), cd.getColor(), cd.isCaseInsensitive(), cd.isHint(), cd.getEntries(), cd.getFalsePositives(), cd.getFalseRecommendations(), cd.isDossierDictionary()))
.collect(Collectors.toList());
}
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -25,10 +25,10 @@ import org.springframework.core.io.ClassPathResource;
import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -27,12 +27,12 @@ import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.google.common.collect.Sets;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.storage.commons.service.StorageService;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -58,11 +58,11 @@ import com.amazonaws.services.s3.AmazonS3;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -36,9 +36,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;