diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index a3250644..ede60e67 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -12,7 +12,7 @@ redaction-service-api-v1 - 2.70.0 + 2.75.0 diff --git a/redaction-service-v1/redaction-service-server-v1/pom.xml b/redaction-service-v1/redaction-service-server-v1/pom.xml index 2f090926..774c2791 100644 --- a/redaction-service-v1/redaction-service-server-v1/pom.xml +++ b/redaction-service-v1/redaction-service-server-v1/pom.xml @@ -27,6 +27,12 @@ spring-boot-starter-aop + + com.iqser.red.commons + dictionary-merge-commons + 1.3.0 + + com.iqser.red.commons storage-commons diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java index a6a7ad77..607157e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java @@ -9,6 +9,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; +import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.multitenancy.AsyncConfig; import com.iqser.red.service.redaction.v1.server.multitenancy.MultiTenancyMessagingConfiguration; @@ -38,4 +39,10 @@ public class Application { return new TimedAspect(registry); } + @Bean + public DictionaryMergeService dictionaryMergeService() { + + return new DictionaryMergeService(); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java index 022fafb4..34b947f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java @@ -48,9 +48,11 @@ public class RedactionMessageReceiver { // This prevents from endless retries oom errors. if (message.getMessageProperties().isRedelivered()) { - throw new AmqpRejectAndDontRequeueException(String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", + var errorMessage = String.format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.", analyzeRequest.getDossierId(), - analyzeRequest.getFileId())); + analyzeRequest.getFileId()); + fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), new FileErrorInfo(errorMessage, priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS))); + throw new AmqpRejectAndDontRequeueException(errorMessage); } receiveAnalyzeRequest(analyzeRequest, priority); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java index 20ece317..ac80349b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java @@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import java.util.HashSet; import java.util.Set; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntryModel.java deleted file mode 100644 index efe978d7..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntryModel.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; - -import lombok.EqualsAndHashCode; - -@EqualsAndHashCode(callSuper = false) -public class DictionaryEntryModel extends DictionaryEntry { - - public DictionaryEntryModel() { - - super(); - } - - - public DictionaryEntryModel(long entryId, String value, long version, boolean deleted, String typeId) { - - super(entryId, value, version, deleted, typeId); - } - - - public DictionaryEntryModel(DictionaryEntry entry) { - - super(entry.getEntryId(), entry.getValue(), entry.getVersion(), entry.isDeleted(), entry.getTypeId()); - } - - - @Override - @EqualsAndHashCode.Include - public String getValue() { - - return super.getValue(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java index c9a64aa5..a6b43116 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java @@ -5,7 +5,8 @@ import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index 39ca5868..40634bdf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -1,12 +1,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import java.awt.Color; +import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -19,13 +18,15 @@ import org.springframework.stereotype.Service; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.iqser.red.service.dictionarymerge.commons.CommonsDictionaryModel; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; +import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; @@ -51,6 +52,8 @@ public class DictionaryService { private final RedactionServiceSettings settings; + private final DictionaryMergeService dictionaryMergeService; + private LoadingCache tenantDictionaryCache; @@ -291,33 +294,24 @@ public class DictionaryService { @Timed("redactmanager_getDeepCopyDictionary") public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { - Map mergedDictionaries = new HashMap<>(); + List mergedDictionaries; var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); - dossierTemplateRepresentation.getDictionary().forEach(dm -> mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm))); + var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary(); // merge dictionaries if they have same names long dossierDictionaryVersion = -1; if (dossierDictionaryExists(dossierId)) { var dossierRepresentation = getDossierDictionary(dossierId); - dossierRepresentation.getDictionary().forEach(dm -> { - if (mergedDictionaries.containsKey(dm.getType())) { - //merge dictionary entries, false positives and false recommendations - Set dossierEntries = dm.getEntries(); - mergedDictionaries.get(dm.getType()).getEntries().removeAll(dossierEntries); - mergedDictionaries.get(dm.getType()).getEntries().addAll(dossierEntries); - mergedDictionaries.get(dm.getType()).getFalsePositives().removeAll(dm.getFalsePositives()); - mergedDictionaries.get(dm.getType()).getFalsePositives().addAll(dm.getFalsePositives()); - mergedDictionaries.get(dm.getType()).getFalseRecommendations().removeAll(dm.getFalseRecommendations()); - mergedDictionaries.get(dm.getType()).getFalseRecommendations().addAll(dm.getFalseRecommendations()); - } else { - mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm)); - } - }); + var dossierDictionaries = dossierRepresentation.getDictionary(); + mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), convertDictionaryModel(dossierDictionaries))); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); + } else { + mergedDictionaries = new ArrayList<>(); + dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm))); } - return new Dictionary(mergedDictionaries.values().stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), + return new Dictionary(mergedDictionaries.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); } @@ -373,4 +367,24 @@ public class DictionaryService { } } + private List convertDictionaryModel(List dictionaries) { + return dictionaries.stream().map(d -> CommonsDictionaryModel.builder() + .type(d.getType()) + .rank(d.getRank()) + .color(d.getColor()) + .caseInsensitive(d.isCaseInsensitive()) + .hint(d.isHint()) + .isDossierDictionary(d.isDossierDictionary()) + .entries(d.getEntries()) + .falsePositives(d.getFalsePositives()) + .falseRecommendations(d.getFalseRecommendations()) + .build()).collect(Collectors.toList()); + } + + private List convertCommonsDictionaryModel(List commonsDictionaries) { + return commonsDictionaries.stream().map(cd -> + new DictionaryModel(cd.getType(), cd.getRank(), cd.getColor(), cd.isCaseInsensitive(), cd.isHint(), cd.getEntries(), cd.getFalsePositives(), cd.getFalseRecommendations(), cd.isDossierDictionary())) + .collect(Collectors.toList()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 76952f37..6ab56b02 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -25,10 +25,10 @@ import org.springframework.core.io.ClassPathResource; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 6905501b..62fae085 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -27,12 +27,12 @@ import org.springframework.context.annotation.Primary; import org.springframework.test.context.junit.jupiter.SpringExtension; import com.google.common.collect.Sets; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; -import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.storage.commons.service.StorageService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 2f7186d9..52e9731e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -58,11 +58,11 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index edcec53c..a2cf1697 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -36,9 +36,9 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.Application; import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;