Pull request #524: RED-6224: Made dictionary cache multitenancy ready

Merge in RED/redaction-service from RED-6224 to master

* commit '339833e2d6f4f85757f7fdb84c2006f17d766f38':
  RED-6224: Fixed pr findings
  RED-6224: Made dictionary cache multitenancy ready

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java

@@ -0,0 +1,15 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class TenantDictionary {
+
+    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,24 +1,44 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import java.awt.Color;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import java.util.ArrayList;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import java.util.Comparator;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import java.util.HashSet;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
-import feign.FeignException;
+import javax.annotation.PostConstruct;
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.springframework.stereotype.Service;
 
-import java.awt.*;
+import com.google.common.cache.CacheBuilder;
-import java.util.List;
+import com.google.common.cache.CacheLoader;
-import java.util.*;
+import com.google.common.cache.LoadingCache;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+
+import feign.FeignException;
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
@@ -28,22 +48,39 @@ public class DictionaryService {
     private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
-    private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
+    private final RedactionServiceSettings settings;
-    private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
+
+    private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
 
 
+    @PostConstruct
+    protected void createCache() {
+
+        tenantDictionaryCache = CacheBuilder.newBuilder()
+                .maximumSize(settings.getDictionaryCacheMaximumSize())
+                .expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
+                .build(new CacheLoader<>() {
+                    public TenantDictionary load(String key) {
+
+                        return new TenantDictionary();
+                    }
+                });
+    }
+
+
+    @SneakyThrows
     @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
         log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
         long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
-        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId);
         if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
         }
 
         long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
-        var dossierDictionary = dictionariesByDossier.get(dossierId);
+        var dossierDictionary = getDossierDictionary(dossierId);
         if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
@@ -52,13 +89,14 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDictionaryIncrements")
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
         DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
-        List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
+        List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -77,8 +115,8 @@ public class DictionaryService {
             });
         });
 
-        if (dictionariesByDossier.containsKey(dossierId)) {
+        if (dossierDictionaryExists(dossierId)) {
-            dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
+            dictionaryModels = getDossierDictionary(dossierId).getDictionary();
             dictionaryModels.forEach(dictionaryModel -> {
                 dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                     if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@@ -102,6 +140,7 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
 
         try {
@@ -115,10 +154,10 @@ public class DictionaryService {
 
                     Optional<DictionaryModel> oldModel;
                     if (dossierId == null) {
-                        var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                        var representation = getDossierTemplateDictionary(dossierTemplateId);
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     } else {
-                        var representation = dictionariesByDossier.get(dossierId);
+                        var representation = getDossierDictionary(dossierId);
                         oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     }
 
@@ -178,9 +217,9 @@ public class DictionaryService {
                 dictionaryRepresentation.setDictionary(dictionary);
 
                 if (dossierId == null) {
-                    dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
+                    addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation);
                 } else {
-                    dictionariesByDossier.put(dossierId, dictionaryRepresentation);
+                    addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation);
                 }
             }
         } catch (FeignException e) {
@@ -219,19 +258,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getColor(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.getColor();
         }
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
+        return getDossierTemplateDictionary(dossierTemplateId).getDefaultColor();
     }
 
 
+    @SneakyThrows
     public boolean isHint(String type, String dossierTemplateId) {
 
-        DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
         if (model != null) {
             return model.isHint();
         }
@@ -239,20 +280,21 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     @Timed("redactmanager_getDeepCopyDictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> copy = new ArrayList<>();
 
-        var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
+        var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
         dossierTemplateRepresentation.getDictionary().forEach(dm -> {
             copy.add(SerializationUtils.clone(dm));
         });
 
         //TODO merge dictionaries if they have same names
         long dossierDictionaryVersion = -1;
-        if (dictionariesByDossier.containsKey(dossierId)) {
+        if (dossierDictionaryExists(dossierId)) {
-            var dossierRepresentation = dictionariesByDossier.get(dossierId);
+            var dossierRepresentation = getDossierDictionary(dossierId);
             dossierRepresentation.getDictionary().forEach(dm -> {
                 copy.add(SerializationUtils.clone(dm));
             });
@@ -264,9 +306,45 @@ public class DictionaryService {
     }
 
 
+    @SneakyThrows
     public float[] getNotRedactedColor(String dossierTemplateId) {
 
-        return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
+        return getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor();
+    }
+
+
+    @SneakyThrows
+    private void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) {
+
+        tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
+    }
+
+
+    @SneakyThrows
+    private void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) {
+
+        tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
+    }
+
+
+    @SneakyThrows
+    private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
+    }
+
+
+    @SneakyThrows
+    private DictionaryRepresentation getDossierDictionary(String dossierId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
+    }
+
+
+    @SneakyThrows
+    private boolean dossierDictionaryExists(String dossierId) {
+
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java

@@ -24,4 +24,8 @@ public class RedactionServiceSettings {
 
     private boolean priorityMode;
 
+    private long dictionaryCacheMaximumSize = 100;
+
+    private int dictionaryCacheExpireAfterAccessDays = 3;
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -63,6 +64,8 @@ public class DictionaryServiceTest {
     @Test
     public void testDictionaryServiceConsistency() {
 
+        TenantContext.setTenantId("redaction");
+
         when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
         when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
                 "#cccccc",

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -240,6 +240,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -213,6 +213,8 @@ public class RedactionIntegrationTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
+
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -245,6 +245,7 @@ public class RulesTest {
     @BeforeEach
     public void stubClients() {
 
+        TenantContext.setTenantId("redaction");
         objectMapper.registerModule(new JavaTimeModule());
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -42,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -114,6 +115,7 @@ public class LiveDataIntegrationTest {
     @BeforeEach
     public void prepareTest() {
 
+        TenantContext.setTenantId("redaction");
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
         when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);