diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java index 8a4d6c09..b4ef8f0d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java @@ -17,10 +17,10 @@ import lombok.NoArgsConstructor; public class DictionaryEntries { @Builder.Default - Set entries = new HashSet<>(); + Set entries = new HashSet<>(); @Builder.Default - Set falsePositives = new HashSet<>(); + Set falsePositives = new HashSet<>(); @Builder.Default - Set falseRecommendations = new HashSet<>(); + Set falseRecommendations = new HashSet<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntryModel.java new file mode 100644 index 00000000..3617efcb --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntryModel.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; + +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = false) +public class DictionaryEntryModel extends DictionaryEntry { + + public DictionaryEntryModel() { + super(); + } + + public DictionaryEntryModel(long entryId, String value, long version, boolean deleted, String typeId) { + + super(entryId, value, version, deleted, typeId); + } + + public DictionaryEntryModel(DictionaryEntry entry) { + super(entry.getEntryId(), entry.getValue(), entry.getVersion(), entry.isDeleted(), entry.getTypeId()); + } + + @Override + @EqualsAndHashCode.Include + public String getValue(){ + return super.getValue(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java index 00b413d9..891d7fcc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java @@ -21,9 +21,9 @@ public class DictionaryModel implements Serializable { private final boolean caseInsensitive; private final boolean hint; private final boolean isDossierDictionary; - private final Set entries; - private final Set falsePositives; - private final Set falseRecommendations; + private final Set entries; + private final Set falsePositives; + private final Set falseRecommendations; private transient SearchImplementation entriesSearch; private transient SearchImplementation falsePositiveSearch; @@ -38,9 +38,9 @@ public class DictionaryModel implements Serializable { float[] color, boolean caseInsensitive, boolean hint, - Set entries, - Set falsePositives, - Set falseRecommendations, + Set entries, + Set falsePositives, + Set falseRecommendations, boolean isDossierDictionary) { this.type = type; @@ -54,9 +54,9 @@ public class DictionaryModel implements Serializable { this.falsePositives = falsePositives; this.falseRecommendations = falseRecommendations; - this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), + this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), caseInsensitive); - this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()), + this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()), caseInsensitive); this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream() .filter(e -> !e.isDeleted()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index f1667527..382216cc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -1,11 +1,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import java.awt.Color; -import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -24,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; @@ -160,9 +162,9 @@ public class DictionaryService { oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty(); } - Set entries = new HashSet<>(); - Set falsePositives = new HashSet<>(); - Set falseRecommendations = new HashSet<>(); + Set entries = new HashSet<>(); + Set falsePositives = new HashSet<>(); + Set falseRecommendations = new HashSet<>(); DictionaryEntries newEntries = getEntries(t.getId(), currentVersion); @@ -232,9 +234,15 @@ public class DictionaryService { var type = dictionaryClient.getDictionaryForType(typeId, fromVersion); - Set entries = type.getEntries() != null ? new HashSet<>(type.getEntries()) : new HashSet<>(); - Set falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries()) : new HashSet<>(); - Set falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries()) : new HashSet<>(); + Set entries = type.getEntries() != null ? new HashSet<>(type.getEntries().stream().map(DictionaryEntryModel::new).collect(Collectors.toSet())) : new HashSet<>(); + Set falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries() + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet())) : new HashSet<>(); + Set falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries() + .stream() + .map(DictionaryEntryModel::new) + .collect(Collectors.toSet())) : new HashSet<>(); if (type.isCaseInsensitive()) { entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT))); @@ -283,24 +291,33 @@ public class DictionaryService { @Timed("redactmanager_getDeepCopyDictionary") public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) { - List copy = new ArrayList<>(); + Map mergedDictionaries = new HashMap<>(); var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId); - dossierTemplateRepresentation.getDictionary().forEach(dm -> { - copy.add(SerializationUtils.clone(dm)); - }); + dossierTemplateRepresentation.getDictionary().forEach(dm -> mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm))); - //TODO merge dictionaries if they have same names + // merge dictionaries if they have same names long dossierDictionaryVersion = -1; if (dossierDictionaryExists(dossierId)) { var dossierRepresentation = getDossierDictionary(dossierId); dossierRepresentation.getDictionary().forEach(dm -> { - copy.add(SerializationUtils.clone(dm)); + if (mergedDictionaries.containsKey(dm.getType())) { + //merge dictionary entries, false positives and false recommendations + Set dossierEntries = dm.getEntries(); + mergedDictionaries.get(dm.getType()).getEntries().removeAll(dossierEntries); + mergedDictionaries.get(dm.getType()).getEntries().addAll(dossierEntries); + mergedDictionaries.get(dm.getType()).getFalsePositives().removeAll(dm.getFalsePositives()); + mergedDictionaries.get(dm.getType()).getFalsePositives().addAll(dm.getFalsePositives()); + mergedDictionaries.get(dm.getType()).getFalseRecommendations().removeAll(dm.getFalseRecommendations()); + mergedDictionaries.get(dm.getType()).getFalseRecommendations().addAll(dm.getFalseRecommendations()); + } else { + mergedDictionaries.put(dm.getType(), SerializationUtils.clone(dm)); + } }); dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion(); } - return new Dictionary(copy.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), + return new Dictionary(mergedDictionaries.values().stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()), DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index 88a3a8cb..026828f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -1,11 +1,14 @@ package com.iqser.red.service.redaction.v1.server; +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -18,23 +21,23 @@ import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.test.context.junit.jupiter.SpringExtension; +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; -import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; + @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) @@ -125,4 +128,99 @@ public class DictionaryServiceTest { } + @Test + public void testMergeDictionaries() { + Set dossierTemplateEntries = new HashSet<>(); + dossierTemplateEntries.add(new DictionaryEntry(1L, "dossierTemplateValue", 1, false, "typeId")); + Set dossierEntries = new HashSet<>(); + dossierEntries.add(new DictionaryEntry(1L, "dossierTemplateValue", 1, true, "typeId")); + + Set result = Sets.intersection(dossierTemplateEntries, dossierEntries); + assertThat(result.size()).isZero(); + + Set dossierTemplateEntries2 = new HashSet<>(); + dossierTemplateEntries2.add(new DictionaryEntryModel(1L, "dossierTemplateValue1", 1, false, "typeId")); + dossierTemplateEntries2.add(new DictionaryEntryModel(2L, "dossierTemplateValue2", 1, false, "typeId")); + dossierTemplateEntries2.add(new DictionaryEntryModel(3L, "dossierTemplateValue3", 1, false, "typeId")); + + Set dossierEntries2 = new HashSet<>(); + dossierEntries2.add(new DictionaryEntryModel(1L, "dossierTemplateValue1", 1, true, "typeId")); + dossierEntries2.add(new DictionaryEntryModel(2L, "dossierTemplateValue2", 1, true, "typeId")); + dossierEntries2.add(new DictionaryEntryModel(4L, "dossierTemplateValue4", 1, false, "typeId2")); + + Set mergedResult = new HashSet<>(dossierTemplateEntries2); + mergedResult.removeAll(dossierEntries2); + mergedResult.addAll(dossierEntries2); + assertThat(mergedResult.size()).isEqualTo(4); + + } + + @Test + public void testDictionary() { + + TenantContext.setTenantId("redaction"); + + when(dictionaryClient.getVersion(anyString())).thenReturn(0L); + when(dictionaryClient.getColors(anyString())).thenReturn(new Colors("dtId", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc", + "#cccccc")); + + var type = "type"; + var dtType = new Type(); + dtType.setType(type); + dtType.setId("type1"); + dtType.setVersion(1L); + dtType.setHexColor("#cccccc"); + dtType.setHasDictionary(true); + dtType.setDossierTemplateId("dtId"); + List dossierTemplateEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 1L, false, "type1")).collect(Collectors.toList()); + dossierTemplateEntries.add(new DictionaryEntry(1, "cc", 2L, false, "type1")); + assertThat(dossierTemplateEntries.size()).isEqualTo(3); + dtType.setEntries(dossierTemplateEntries); + + var dossierType = new Type(); + dossierType.setType(type); + dossierType.setId("dossierType"); + dossierType.setVersion(1L); + dossierType.setHexColor("#cccccc"); + dossierType.setHasDictionary(true); + dossierType.setDossierTemplateId("dtId"); + dossierType.setDossierId("dossierId"); + List dossierEntries = Stream.of("aa", "bb").map(t -> new DictionaryEntry(1, t, 2L, true, "dossierType")).collect(Collectors.toList()); + dossierEntries.add(new DictionaryEntry(1, "dd", 1L, false, "dossierType")); + assertThat(dossierEntries.size()).isEqualTo(3); + dossierType.setEntries(dossierEntries); + + when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(dtType)); + when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(List.of(dossierType)); + + when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(dtType); + when(dictionaryClient.getDictionaryForType("dossierType", null)).thenReturn(dossierType); + when(dictionaryClient.getDictionaryForType("type1", 0L)).thenReturn(dtType); + when(dictionaryClient.getDictionaryForType("dossierType", 0L)).thenReturn(dossierType); + + dictionaryService.updateDictionary("dtId", "dossierId"); + var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId"); + assertThat(dict.getDictionaryModels().size()).isEqualTo(1); + var dictModel = dict.getDictionaryModels().get(0); + assertThat(dictModel.getType()).isEqualTo(type); + assertThat(dictModel.getEntries().size()).isEqualTo(4); + dictModel.getEntries().forEach(entry -> { + switch (entry.getValue()) { + case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId()); + case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()); + } + }); + } + }