Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6224: Made dictionary cache multitenancy ready

Browse Source
This commit is contained in:
deiflaender 2023-03-17 13:54:22 +01:00
parent 4b42c8d13f
commit ec0de5b6a2
7 changed files with 107 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class TenantDictionary {
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
}

111
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,25 +1,46 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.awt.Color;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
import feign.FeignException;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.springframework.stereotype.Service;
import java.awt.*;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Service
@RequiredArgsConstructor
@ -28,22 +49,40 @@ public class DictionaryService {
private static final String DEFAULT_COLOR = "#cccccc";
private final DictionaryClient dictionaryClient;
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
@Value("${multitenancy.dictionary-cache.maximumSize:100}")
private Long maximumSize;
@Value("${multitenancy.dictionary-cache.expireAfterAccess:3}")
private Integer expireAfterAccess;
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
@PostConstruct
protected void createCache() {
tenantDictionaryCache = CacheBuilder.newBuilder().maximumSize(maximumSize).expireAfterAccess(expireAfterAccess, TimeUnit.DAYS).build(new CacheLoader<>() {
public TenantDictionary load(String key) {
return new TenantDictionary();
}
});
}
@SneakyThrows
@Timed("redactmanager_updateDictionary")
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
var dossierTemplateDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
}
long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
var dossierDictionary = dictionariesByDossier.get(dossierId);
var dossierDictionary = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
}
@ -52,13 +91,14 @@ public class DictionaryService {
}
@SneakyThrows
@Timed("redactmanager_getDictionaryIncrements")
public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
Set<DictionaryIncrementValue> newValues = new HashSet<>();
List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
List<DictionaryModel> dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@ -77,8 +117,8 @@ public class DictionaryService {
});
});
if (dictionariesByDossier.containsKey(dossierId)) {
dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
dictionaryModels = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@ -102,6 +142,7 @@ public class DictionaryService {
}
@SneakyThrows
private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
try {
@ -115,10 +156,20 @@ public class DictionaryService {
Optional<DictionaryModel> oldModel;
if (dossierId == null) {
var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
DictionaryRepresentation representation = null;
try {
representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
} catch (ExecutionException e) {
throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
}
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
} else {
var representation = dictionariesByDossier.get(dossierId);
DictionaryRepresentation representation = null;
try {
representation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
} catch (ExecutionException e) {
throw new RuntimeException("Failed to load Dictionary cache for tenant: " + TenantContext.getTenantId());
}
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
}
@ -178,9 +229,9 @@ public class DictionaryService {
dictionaryRepresentation.setDictionary(dictionary);
if (dossierId == null) {
dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
} else {
dictionariesByDossier.put(dossierId, dictionaryRepresentation);
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
}
}
} catch (FeignException e) {
@ -219,19 +270,21 @@ public class DictionaryService {
}
@SneakyThrows
public float[] getColor(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) {
return model.getColor();
}
return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getDefaultColor();
}
@SneakyThrows
public boolean isHint(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
DictionaryModel model = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) {
return model.isHint();
}
@ -239,20 +292,21 @@ public class DictionaryService {
}
@SneakyThrows
@Timed("redactmanager_getDeepCopyDictionary")
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
List<DictionaryModel> copy = new ArrayList<>();
var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
var dossierTemplateRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
dossierTemplateRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm));
});
//TODO merge dictionaries if they have same names
long dossierDictionaryVersion = -1;
if (dictionariesByDossier.containsKey(dossierId)) {
var dossierRepresentation = dictionariesByDossier.get(dossierId);
if (tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId)) {
var dossierRepresentation = tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
dossierRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm));
});
@ -264,9 +318,10 @@ public class DictionaryService {
}
@SneakyThrows
public float[] getNotRedactedColor(String dossierTemplateId) {
return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId).getNotRedactedColor();
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
@ -63,6 +64,8 @@ public class DictionaryServiceTest {
@Test
public void testDictionaryServiceConsistency() {
TenantContext.setTenantId("redaction");
when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
"#cccccc",

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java
View File

@ -240,6 +240,7 @@ public class HeadlinesGoldStandardIntegrationTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -213,6 +213,8 @@ public class RedactionIntegrationTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -245,6 +245,7 @@ public class RulesTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
objectMapper.registerModule(new JavaTimeModule());
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -42,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@ -114,6 +115,7 @@ public class LiveDataIntegrationTest {
@BeforeEach
public void prepareTest() {
TenantContext.setTenantId("redaction");
when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);