test all entries in dict

processed date fix
Pull request #349 : RED-3611: Added priority mode that only listens to priority queue
2022-03-19 21:44:32 +02:00 · 2022-03-19 19:45:10 +02:00 · 2022-03-16 10:22:22 +01:00 · 2022-03-16 09:56:54 +01:00 · 2022-03-15 18:51:02 +01:00 · 2022-03-15 17:45:26 +01:00
10 changed files with 209 additions and 129 deletions
@@ -12,7 +12,7 @@
    <artifactId>redaction-service-api-v1</artifactId>

    <properties>
-        <persistence-service.version>1.39.0</persistence-service.version>
+        <persistence-service.version>1.70.2</persistence-service.version>
    </properties>

    <dependencies>
@@ -169,7 +169,7 @@ public class RedactionController implements RedactionResource {
    @Override
    public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {

-        log.info("Requested preview for: {}", redactionRequest);
+        log.debug("Requested preview for: {}", redactionRequest);
        dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());

        var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class MessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+        redactionMessageReceiver.receiveAnalyzeRequest(in, false);
+    }
+
+}
@@ -1,8 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;

 import lombok.RequiredArgsConstructor;
+
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

@@ -10,11 +12,24 @@ import org.springframework.context.annotation.Configuration;
@RequiredArgsConstructor
 public class MessagingConfiguration {

-
    public static final String REDACTION_QUEUE = "redactionQueue";
-
    public static final String REDACTION_DQL = "redactionDQL";

+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
+    public MessageReceiver messageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new MessageReceiver(redactionMessageReceiver);
+    }
+
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+    public PriorityMessageReceiver priorityMessageReceiver(RedactionMessageReceiver redactionMessageReceiver){
+        return new PriorityMessageReceiver(redactionMessageReceiver);
+    }
+

    @Bean
    public Queue redactionQueue() {
@@ -27,9 +42,21 @@ public class MessagingConfiguration {
    }


+    @Bean
+    public Queue redactionPriorityQueue() {
+
+        return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
+                .withArgument("x-dead-letter-exchange", "")
+                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
+                .maxPriority(2)
+                .build();
+    }
+
+
    @Bean
    public Queue redactionDeadLetterQueue() {

        return QueueBuilder.durable(REDACTION_DQL).build();
    }
+
 }
@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class PriorityMessageReceiver {
+
+    private final RedactionMessageReceiver redactionMessageReceiver;
+
+
+    @RabbitHandler
+    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE)
+    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+
+        redactionMessageReceiver.receiveAnalyzeRequest(in, true);
+    }
+
+}
@@ -30,12 +30,10 @@ public class RedactionMessageReceiver {
    private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;


-    @RabbitHandler
-    @RabbitListener(queues = REDACTION_QUEUE)
-    public void receiveAnalyzeRequest(String in) throws JsonProcessingException {
+    public void receiveAnalyzeRequest(String in, boolean priority) throws JsonProcessingException {

        var analyzeRequest = objectMapper.readValue(in, AnalyzeRequest.class);
-        log.info("Processing analyze request for file: {}", analyzeRequest.getFileId());
+        log.info("Processing priority: {} analyze request for file: {}", priority, analyzeRequest.getFileId());
        AnalyzeResult result = null;

        switch (analyzeRequest.getMessageType()) {
@@ -5,7 +5,6 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import feign.FeignException;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -36,13 +35,13 @@ public class DictionaryService {
        long dossierTemplateDictionaryVersion = dictionaryClient.getVersion(dossierTemplateId);
        var dossierTemplateDictionary = dictionariesByDossierTemplate.get(dossierTemplateId);
        if (dossierTemplateDictionary == null || dossierTemplateDictionaryVersion > dossierTemplateDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, null);
+            updateDictionaryEntry(dossierTemplateId, dossierTemplateDictionaryVersion, getVersion(dossierTemplateDictionary), null);
        }

        long dossierDictionaryVersion = dictionaryClient.getVersionForDossier(dossierId);
        var dossierDictionary = dictionariesByDossier.get(dossierId);
        if (dossierDictionary == null || dossierDictionaryVersion > dossierDictionary.getDictionaryVersion()) {
-            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, dossierId);
+            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
        }

        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
@@ -78,7 +77,7 @@ public class DictionaryService {
    }


-    private void updateDictionaryEntry(String dossierTemplateId, long version, String dossierId) {
+    private void updateDictionaryEntry(String dossierTemplateId, long version, Long currentVersion, String dossierId) {

        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
@@ -88,8 +87,31 @@ public class DictionaryService {

                List<DictionaryModel> dictionary = typeResponse
                        .stream()
-                        .map(t -> new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
-                                .isHint(), t.isRecommendation(), convertEntries(t.getId()), new HashSet<>(), dossierId != null))
+                        .map(t -> {
+
+                            Optional<DictionaryModel> oldModel;
+                            if (dossierId == null) {
+                                var representation = dictionariesByDossierTemplate.get(dossierTemplateId);
+                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                            } else {
+                                var representation = dictionariesByDossier.get(dossierId);
+                                oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                            }
+
+                            Set<DictionaryEntry> entries = new HashSet<>();
+                            var newEntries = convertEntries(t.getId(), currentVersion);
+                            var newValues = newEntries.stream().map(v -> v.getValue()).collect(Collectors.toSet());
+
+                            // add old entries from existing DictionaryModel
+                            oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(
+                                    f -> !newValues.contains(f.getValue())).collect(Collectors.toList())
+                            ));
+                            // Add Increments
+                            entries.addAll(convertEntries(t.getId(), currentVersion));
+
+                            return new DictionaryModel(t.getType(), t.getRank(), convertColor(t.getHexColor()), t.isCaseInsensitive(), t
+                                    .isHint(), t.isRecommendation(), entries, new HashSet<>(), dossierId != null);
+                        })
                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                        .collect(Collectors.toList());

@@ -118,9 +140,9 @@ public class DictionaryService {
    }


-    private Set<DictionaryEntry> convertEntries(String typeId) {
+    private Set<DictionaryEntry> convertEntries(String typeId, Long fromVersion) {

-        var type = dictionaryClient.getDictionaryForType(typeId);
+        var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);

        Set<DictionaryEntry> entries = new HashSet<>(type
                .getEntries());
@@ -219,4 +241,11 @@ public class DictionaryService {
        return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
    }

+    private Long getVersion(DictionaryRepresentation dictionaryRepresentation) {
+        if (dictionaryRepresentation == null) {
+            return null;
+        } else {
+            return dictionaryRepresentation.getDictionaryVersion();
+        }
+    }
 }
@@ -277,17 +277,15 @@ public class RedactionLogMergeService {

        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {

-            if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), dossierTemplateId);
+            redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
+            redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId()));
+            redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
+            redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());

-                sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);
+            sectionTextService.handleSectionText(sectionGrid, redactionLogEntry);

-                redactionLogEntries.add(redactionLogEntry);
-            }
+            redactionLogEntries.add(redactionLogEntry);
        }

        return redactionLogEntries;
@@ -302,15 +300,6 @@ public class RedactionLogMergeService {
                .collect(Collectors.toList());
    }

-
-    private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
-
-        return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
-                manualRedactionEntry.getProcessedDate() != null &&
-                (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
-    }
-
-
    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
                                                      String dossierTemplateId) {

@@ -20,4 +20,6 @@ public class RedactionServiceSettings {

    private boolean nerServiceEnabled = true;

+    private boolean priorityMode;
+
 }
@@ -1,28 +1,33 @@
 package com.iqser.red.service.redaction.v1.server;

-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.when;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
+import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.*;
+import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
+import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import lombok.SneakyThrows;
 import org.apache.commons.io.IOUtils;
 import org.junit.After;
 import org.junit.Before;
@@ -47,48 +52,15 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;

-import com.amazonaws.services.s3.AmazonS3;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
-import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
-import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.model.RedactionRequest;
-import com.iqser.red.service.redaction.v1.model.RedactionResult;
-import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
-import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
-import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
+import java.io.*;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.*;
+import java.util.stream.Collectors;

-import lombok.SneakyThrows;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -234,31 +206,39 @@ public class RedactionIntegrationTest {
                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
                .build()));

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
-        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
-        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
-        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false));
-        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
-        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
-        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
-        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PURITY, false));
-        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false));
-        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false));
-        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false));
-        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
-        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
    }

+    
+    private void mockDictionaryCalls(Long version){
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID,version)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
+
+    }

    @Test
    public void test270Rotated() {
@@ -519,9 +499,9 @@ public class RedactionIntegrationTest {

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));

-        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, null)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));

        start = System.currentTimeMillis();

@@ -557,7 +537,7 @@ public class RedactionIntegrationTest {

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));

        analyzeService.reanalyze(request);

@@ -582,17 +562,18 @@ public class RedactionIntegrationTest {
        dictionary.get(AUTHOR).add("report");
        reanlysisVersions.put("report", 2L);
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        mockDictionaryCalls(0L);
+
        analyzeService.reanalyze(request);

        dictionary.get(AUTHOR).add("assessment report");
        reanlysisVersions.put("assessment report", 3L);
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        mockDictionaryCalls(2L);
        analyzeService.reanalyze(request);

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
-        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
+        mockDictionaryCalls(3L);
        analyzeService.reanalyze(request);

        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
@@ -676,6 +657,7 @@ public class RedactionIntegrationTest {

        dictionary.get(AUTHOR).add("properties");
        reanlysisVersions.put("properties", 1L);
+        mockDictionaryCalls(0L);

        dictionary.get(AUTHOR).add("physical");
        reanlysisVersions.put("physical", 2L);
@@ -690,9 +672,9 @@ public class RedactionIntegrationTest {

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));

-        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
+        when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, null)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));

        start = System.currentTimeMillis();

@@ -725,10 +707,11 @@ public class RedactionIntegrationTest {

        deleted.remove("mouse");
        reanlysisVersions.put("mouse", 4L);
+        mockDictionaryCalls(3L);

        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);

-        when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE, null)).thenReturn(getDictionaryResponse(VERTEBRATE, false));

        analyzeService.reanalyze(request);
Author	SHA1	Message	Date
Timo Bejan	5db6db52e5	test all entries in dict	2022-03-19 21:44:32 +02:00
Timo Bejan	c1317ee521	processed date fix	2022-03-19 19:45:10 +02:00
Dominique Eiflaender	4b73d3470a	Pull request #349 : RED-3611: Added priority mode that only listens to priority queue Merge in RED/redaction-service from RED-3611 to release/3.80.x * commit '19014c683e12944e23c1afccbf133530c6548d90': RED-3611: Added priority mode that only listens to priority queue	2022-03-16 10:22:22 +01:00
deiflaender	19014c683e	RED-3611: Added priority mode that only listens to priority queue	2022-03-16 09:56:54 +01:00
Dominique Eiflaender	1d25575e24	Pull request #348 : RED-3620: Fixed remove from dictionary Merge in RED/redaction-service from RED-3620 to release/3.80.x * commit '358395e69b0b6595f706186971eb6580924fe7d5': RED-3620: Fixed remove from dictionary	2022-03-15 18:51:02 +01:00
deiflaender	358395e69b	RED-3620: Fixed remove from dictionary	2022-03-15 17:45:26 +01:00
deiflaender	34dc4f4a98	Changed log message to debug	2022-03-14 15:13:41 +01:00
deiflaender	f5f84948ee	Fixed unittests	2022-03-14 14:06:49 +01:00
Timo Bejan	2cfa603f85	Adjusted tests	2022-03-14 14:18:15 +02:00
Timo Bejan	4da5252bae	Adjusted tests to new API	2022-03-14 13:54:02 +02:00
Timo Bejan	2f6841a1d2	Adjusted tests to new API	2022-03-14 13:48:07 +02:00
Timo Bejan	a896c51335	DictionaryModel Incremental build	2022-03-14 13:42:43 +02:00
Timo Bejan	47a45e9650	Release 3.80.x branch creation + incremental entry loading based on version for dictionaries	2022-03-14 13:33:27 +02:00