From ec0de5b6a2320ace498d29411ffe65ca7d0d8035 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 17 Mar 2023 13:54:22 +0100 Subject: [PATCH] RED-6224: Made dictionary cache multitenancy ready --- .../redaction/model/TenantDictionary.java | 15 +++ .../redaction/service/DictionaryService.java | 111 +++++++++++++----- .../v1/server/DictionaryServiceTest.java | 3 + .../HeadlinesGoldStandardIntegrationTest.java | 1 + .../v1/server/RedactionIntegrationTest.java | 2 + .../redaction/v1/server/RulesTest.java | 1 + .../realdata/LiveDataIntegrationTest.java | 2 + 7 files changed, 107 insertions(+), 28 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java new file mode 100644 index 00000000..d3f3b7d6 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java @@ -0,0 +1,15 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class TenantDictionary { + + private final Map dictionariesByDossierTemplate = new HashMap<>(); + private final Map dictionariesByDossier = new HashMap<>(); +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index f8ce0c1f..c0308ef6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -1,25 +1,46 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.awt.Color; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.SerializationUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary; import feign.FeignException; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.SerializationUtils; -import org.springframework.stereotype.Service; - -import java.awt.*; -import java.util.List; -import java.util.*; -import java.util.stream.Collectors; - @Slf4j @Service @RequiredArgsConstructor @@ -28,22 +49,40 @@ public class DictionaryService { private static final String DEFAULT_COLOR = "#cccccc"; private final DictionaryClient dictionaryClient; - private final Map dictionariesByDossierTemplate = new HashMap<>(); - private final Map dictionariesByDossier = new HashMap<>(); + @Value("${multitenancy.dictionary-cache.maximumSize:100}") + private Long maximumSize; + + @Value("${multitenancy.dictionary-cache.expireAfterAccess:3}") + private Integer expireAfterAccess; + + private LoadingCache tenantDictionaryCache; + @PostConstruct + protected void createCache() { + + tenantDictionaryCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(expireAfterAccess, TimeUnit.DAYS).build(new CacheLoader<>() { + public TenantDictionary load(String key) { + + return new TenantDictionary(); + } + }); + } + + + @SneakyThrows @Timed("redactmanager_updateDictionary") public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) { log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId); long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId); - var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId); + var dossierTemplateDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId); if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) { updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null); } long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId); - var dossierDictionary = dictionariesByDossier.get(dossierId); + var dossierDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId); if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) { updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId); } @@ -52,13 +91,14 @@ public class DictionaryService { } + @SneakyThrows @Timed("redactmanager_getDictionaryIncrements") public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) { DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId); Set newValues = new HashSet<>(); - List dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary(); + List dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> { if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) { @@ -77,8 +117,8 @@ public class DictionaryService { }); }); - if (dictionariesByDossier.containsKey(dossierId)) { - dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary(); + if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) { + dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> { if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) { @@ -102,6 +142,7 @@ public class DictionaryService { } + @SneakyThrows private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) { try { @@ -115,10 +156,20 @@ public class DictionaryService { Optional oldModel; if (dossierId == null) { - var representation = dictionariesByDossierTemplate.get(dossierTemplateId); + DictionaryRepresentation representation = null; + try { + representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId); + } catch (ExecutionException e) { + throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId()); + } oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty(); } else { - var representation = dictionariesByDossier.get(dossierId); + DictionaryRepresentation representation = null; + try { + representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId); + } catch (ExecutionException e) { + throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId()); + } oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty(); } @@ -178,9 +229,9 @@ public class DictionaryService { dictionaryRepresentation.setDictionary(dictionary); if (dossierId == null) { - dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation); + tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation); } else { - dictionariesByDossier.put(dossierId, dictionaryRepresentation); + tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation); } } } catch (FeignException e) { @@ -219,19 +270,21 @@ public class DictionaryService { } + @SneakyThrows public float[] getColor(String type, String dossierTemplateId) { - DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type); + DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.getColor(); } - return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor(); + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDefaultColor(); } + @SneakyThrows public boolean isHint(String type, String dossierTemplateId) { - DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type); + DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type); if (model != null) { return model.isHint(); } @@ -239,20 +292,21 @@ public class DictionaryService { } + @SneakyThrows @Timed("redactmanager_getDeepCopyDictionary") public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { List copy = new ArrayList<>(); - var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId); + var dossierTemplateRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId); dossierTemplateRepresentation.getDictionary().forEach(dm -> { copy.add(SerializationUtils.clone(dm)); }); //TODO merge dictionaries if they have same names long dossierDictionaryVersion = -1; - if (dictionariesByDossier.containsKey(dossierId)) { - var dossierRepresentation = dictionariesByDossier.get(dossierId); + if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) { + var dossierRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId); dossierRepresentation.getDictionary().forEach(dm -> { copy.add(SerializationUtils.clone(dm)); }); @@ -264,9 +318,10 @@ public class DictionaryService { } + @SneakyThrows public float[] getNotRedactedColor(String dossierTemplateId) { - return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor(); + return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getNotRedactedColor(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 52b434e8..fed7ce46 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.storage.commons.StorageAutoConfiguration; @@ -63,6 +64,8 @@ public class DictionaryServiceTest { @Test public void testDictionaryServiceConsistency() { + TenantContext.setTenantId("redaction"); + when(dictionaryClient.getVersion(anyString())).thenReturn(0L); when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId", "#cccccc", diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java index 69052f68..bc0fe03d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java @@ -240,6 +240,7 @@ public class HeadlinesGoldStandardIntegrationTest { @BeforeEach public void stubClients() { + TenantContext.setTenantId("redaction"); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 9bdbb63d..e5d61ddb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -213,6 +213,8 @@ public class RedactionIntegrationTest { @BeforeEach public void stubClients() { + TenantContext.setTenantId("redaction"); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index b453453c..d4d54446 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -245,6 +245,7 @@ public class RulesTest { @BeforeEach public void stubClients() { + TenantContext.setTenantId("redaction"); objectMapper.registerModule(new JavaTimeModule()); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java index 3f049cf9..070c25cb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java @@ -42,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; @@ -114,6 +115,7 @@ public class LiveDataIntegrationTest { @BeforeEach public void prepareTest() { + TenantContext.setTenantId("redaction"); when(dictionaryClient.getVersion(anyString())).thenReturn(1L); when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);