Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #524: RED-6224: Made dictionary cache multitenancy ready

Browse Source 
Merge in RED/redaction-service from RED-6224 to master

* commit '339833e2d6f4f85757f7fdb84c2006f17d766f38':
  RED-6224: Fixed pr findings
  RED-6224: Made dictionary cache multitenancy ready
This commit is contained in:
Dominique Eiflaender 2023-03-17 16:17:57 +01:00
commit cec18a472f
8 changed files with 137 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java Normal file
View File

@ -0,0 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.HashMap;
import java.util.Map;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class TenantDictionary {
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
}

140
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -1,24 +1,44 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import java.awt.Color;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import feign.FeignException;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.PostConstruct;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.SerializationUtils;
import org.springframework.stereotype.Service;
import java.awt.*;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import feign.FeignException;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@ -28,22 +48,39 @@ public class DictionaryService {
private static final String DEFAULT_COLOR = "#cccccc";
private final DictionaryClient dictionaryClient;
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
private final RedactionServiceSettings settings;
private LoadingCache<String, TenantDictionary> tenantDictionaryCache;
@PostConstruct
protected void createCache() {
tenantDictionaryCache = CacheBuilder.newBuilder()
.maximumSize(settings.getDictionaryCacheMaximumSize())
.expireAfterAccess(settings.getDictionaryCacheExpireAfterAccessDays(), TimeUnit.DAYS)
.build(new CacheLoader<>() {
public TenantDictionary load(String key) {
return new TenantDictionary();
}
});
}
@SneakyThrows
@Timed("redactmanager_updateDictionary")
public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
log.info("Updating dictionary data for dossierTemplate {} and dossier {}", dossierTemplateId, dossierId);
long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
var dossierTemplateDictionary = getDossierTemplateDictionary(dossierTemplateId);
if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
}
long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
var dossierDictionary = dictionariesByDossier.get(dossierId);
var dossierDictionary = getDossierDictionary(dossierId);
if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
}
@ -52,13 +89,14 @@ public class DictionaryService {
}
@SneakyThrows
@Timed("redactmanager_getDictionaryIncrements")
public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
DictionaryVersion version = updateDictionary(dossierTemplateId, dossierId);
Set<DictionaryIncrementValue> newValues = new HashSet<>();
List<DictionaryModel> dictionaryModels = dictionariesByDossierTemplate.get(dossierTemplateId).getDictionary();
List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@ -77,8 +115,8 @@ public class DictionaryService {
});
});
if (dictionariesByDossier.containsKey(dossierId)) {
dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
if (dossierDictionaryExists(dossierId)) {
dictionaryModels = getDossierDictionary(dossierId).getDictionary();
dictionaryModels.forEach(dictionaryModel -> {
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
@ -102,6 +140,7 @@ public class DictionaryService {
}
@SneakyThrows
private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {
try {
@ -115,10 +154,10 @@ public class DictionaryService {
Optional<DictionaryModel> oldModel;
if (dossierId == null) {
var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
var representation = getDossierTemplateDictionary(dossierTemplateId);
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
} else {
var representation = dictionariesByDossier.get(dossierId);
var representation = getDossierDictionary(dossierId);
oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
}
@ -178,9 +217,9 @@ public class DictionaryService {
dictionaryRepresentation.setDictionary(dictionary);
if (dossierId == null) {
dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
addDictionaryRepresentationForDossierTemplate(dossierTemplateId, dictionaryRepresentation);
} else {
dictionariesByDossier.put(dossierId, dictionaryRepresentation);
addDictionaryRepresentationForDossier(dossierId, dictionaryRepresentation);
}
}
} catch (FeignException e) {
@ -219,19 +258,21 @@ public class DictionaryService {
}
@SneakyThrows
public float[] getColor(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) {
return model.getColor();
}
return dictionariesByDossierTemplate.get(dossierTemplateId).getDefaultColor();
return getDossierTemplateDictionary(dossierTemplateId).getDefaultColor();
}
@SneakyThrows
public boolean isHint(String type, String dossierTemplateId) {
DictionaryModel model = dictionariesByDossierTemplate.get(dossierTemplateId).getLocalAccessMap().get(type);
DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
if (model != null) {
return model.isHint();
}
@ -239,20 +280,21 @@ public class DictionaryService {
}
@SneakyThrows
@Timed("redactmanager_getDeepCopyDictionary")
public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
List<DictionaryModel> copy = new ArrayList<>();
var dossierTemplateRepresentation = dictionariesByDossierTemplate.get(dossierTemplateId);
var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
dossierTemplateRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm));
});
//TODO merge dictionaries if they have same names
long dossierDictionaryVersion = -1;
if (dictionariesByDossier.containsKey(dossierId)) {
var dossierRepresentation = dictionariesByDossier.get(dossierId);
if (dossierDictionaryExists(dossierId)) {
var dossierRepresentation = getDossierDictionary(dossierId);
dossierRepresentation.getDictionary().forEach(dm -> {
copy.add(SerializationUtils.clone(dm));
});
@ -264,9 +306,45 @@ public class DictionaryService {
}
@SneakyThrows
public float[] getNotRedactedColor(String dossierTemplateId) {
return dictionariesByDossierTemplate.get(dossierTemplateId).getNotRedactedColor();
return getDossierTemplateDictionary(dossierTemplateId).getNotRedactedColor();
}
@SneakyThrows
private void addDictionaryRepresentationForDossierTemplate(String dossierTemplateId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().put(dossierTemplateId, dictionaryRepresentation);
}
@SneakyThrows
private void addDictionaryRepresentationForDossier(String dossierId, DictionaryRepresentation dictionaryRepresentation) {
tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().put(dossierId, dictionaryRepresentation);
}
@SneakyThrows
private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
}
@SneakyThrows
private DictionaryRepresentation getDossierDictionary(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
}
@SneakyThrows
private boolean dossierDictionaryExists(String dossierId) {
return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().containsKey(dossierId);
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/settings/RedactionServiceSettings.java
View File

@ -24,4 +24,8 @@ public class RedactionServiceSettings {
private boolean priorityMode;
private long dictionaryCacheMaximumSize = 100;
private int dictionaryCacheExpireAfterAccessDays = 3;
}

3
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
@ -63,6 +64,8 @@ public class DictionaryServiceTest {
@Test
public void testDictionaryServiceConsistency() {
TenantContext.setTenantId("redaction");
when(dictionaryClient.getVersion(anyString())).thenReturn(0L);
when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dossierTemplateId",
"#cccccc",

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java
View File

@ -240,6 +240,7 @@ public class HeadlinesGoldStandardIntegrationTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -213,6 +213,8 @@ public class RedactionIntegrationTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -245,6 +245,7 @@ public class RulesTest {
@BeforeEach
public void stubClients() {
TenantContext.setTenantId("redaction");
objectMapper.registerModule(new JavaTimeModule());
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -42,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@ -114,6 +115,7 @@ public class LiveDataIntegrationTest {
@BeforeEach
public void prepareTest() {
TenantContext.setTenantId("redaction");
when(dictionaryClient.getVersion(anyString())).thenReturn(1L);
when(dictionaryClient.getVersionForDossier(anyString())).thenReturn(1L);