Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9123: Improve performance of re-analysis (Spike)

Browse Source
This commit is contained in:
Maverick Studer 2024-07-02 12:52:24 +02:00
parent 3e75f3e5e7
commit 776a24306f
17 changed files with 187 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
}
description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.467.0"
val persistenceServiceVersion = "2.473.0"
dependencies {
implementation("org.springframework:spring-web:6.0.12")

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.467.0"
val persistenceServiceVersion = "2.473.0"
val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4"
val testContainersVersion = "1.19.7"

25
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java
View File

@ -23,6 +23,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -97,8 +98,7 @@ public class DictionaryService {
updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
}
return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion)
.build();
return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
}
@ -167,13 +167,18 @@ public class DictionaryService {
try {
DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId,
true);
var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, currentVersion, true) : dictionaryClient.getAllTypesForDossier(
dossierId,
currentVersion,
true);
if (CollectionUtils.isNotEmpty(typeResponse)) {
String tenantId = TenantContext.getTenantId();
List<DictionaryModel> dictionary = typeResponse.stream()
.parallel()
.map(t -> {
TenantContext.setTenantId(tenantId);
Optional<DictionaryModel> optionalOldModel;
if (dossierId == null) {
var representation = getDossierTemplateDictionary(dossierTemplateId);
@ -193,7 +198,7 @@ public class DictionaryService {
Set<DictionaryEntryModel> falsePositives = new HashSet<>();
Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
DictionaryEntries newEntries = mapEntries(t);
var newValues = newEntries.getEntries()
.stream()
@ -308,9 +313,7 @@ public class DictionaryService {
}
private DictionaryEntries getEntries(String typeId, Long fromVersion) {
var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);
private DictionaryEntries mapEntries(Type type) {
Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries()
.stream()
@ -334,7 +337,7 @@ public class DictionaryService {
entries.size(),
falsePositives.size(),
falseRecommendations.size(),
typeId);
entries);
return new DictionaryEntries(entries, falsePositives, falseRecommendations);
}
@ -400,7 +403,9 @@ public class DictionaryService {
return new Dictionary(mergedDictionaries.stream()
.sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
.collect(Collectors.toList()),
DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion)
DictionaryVersion.builder()
.dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
.dossierVersion(dossierDictionaryVersion)
.build());
}

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -11,6 +11,7 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
@ -47,6 +48,8 @@ public class RedactionStorageService {
private final EntityLogMongoService entityLogMongoService;
private final TaskExecutor taskExecutor;
@SneakyThrows
public InputStream getStoredObject(String storageId) {
@ -228,24 +231,24 @@ public class RedactionStorageService {
Supplier<DocumentStructure> documentStructureSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
DocumentStructure.class);
CompletableFuture<DocumentStructure> documentStructureFuture = CompletableFuture.supplyAsync(documentStructureSupplier);
CompletableFuture<DocumentStructure> documentStructureFuture = CompletableFuture.supplyAsync(documentStructureSupplier, taskExecutor);
Supplier<DocumentTextData[]> documentTextDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
DocumentTextData[].class);
CompletableFuture<DocumentTextData[]> documentTextDataFuture = CompletableFuture.supplyAsync(documentTextDataSupplier);
CompletableFuture<DocumentTextData[]> documentTextDataFuture = CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
Supplier<DocumentPositionData[]> documentPositionDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId,
fileId,
FileType.DOCUMENT_POSITION),
DocumentPositionData[].class);
CompletableFuture<DocumentPositionData[]> documentPositionDataFuture = CompletableFuture.supplyAsync(documentPositionDataSupplier);
CompletableFuture<DocumentPositionData[]> documentPositionDataFuture = CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
Supplier<DocumentPage[]> documentPageSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
DocumentPage[].class);
CompletableFuture<DocumentPage[]> documentPagesFuture = CompletableFuture.supplyAsync(documentPageSupplier);
CompletableFuture<DocumentPage[]> documentPagesFuture = CompletableFuture.supplyAsync(documentPageSupplier, taskExecutor);
CompletableFuture.allOf(documentStructureFuture, documentTextDataFuture, documentPositionDataFuture, documentPagesFuture).join();

61
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -186,7 +186,6 @@ public abstract class AbstractRedactionIntegrationTest {
protected final Map<String, Long> reanlysisVersions = new HashMap<>();
protected final Set<String> deleted = new HashSet<>();
@MockBean
protected RulesClient rulesClient;
@ -260,6 +259,9 @@ public abstract class AbstractRedactionIntegrationTest {
true));
when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true));
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, version, true)).then((Answer<List<Type>>)invocation -> (getTypeResponseForTemplate()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, version, true)).then((Answer<List<Type>>)invocation -> ((getTypeResponseForDossier())));
}
@ -513,7 +515,7 @@ public abstract class AbstractRedactionIntegrationTest {
}
protected List<Type> getTypeResponse() {
protected List<Type> getTypeResponseForTemplate() {
return typeColorMap.entrySet()
.stream()
@ -526,9 +528,62 @@ public abstract class AbstractRedactionIntegrationTest {
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
.rank(rankTypeMap.get(typeColor.getKey()))
.entries(toDictionaryEntry(dictionary.get(typeColor.getKey())))
.build())
.toList();
}
.collect(Collectors.toList());
protected List<Type> getTypeResponseForDossier() {
return List.of(Type.builder()
.id(IMPORTED_REDACTION_TYPE_ID)
.type(IMPORTED_REDACTION_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(IMPORTED_REDACTION_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(IMPORTED_REDACTION_INDICATOR))
.isRecommendation(recommendationTypeMap.get(IMPORTED_REDACTION_INDICATOR))
.rank(rankTypeMap.get(IMPORTED_REDACTION_INDICATOR))
.entries(toDictionaryEntry(dossierDictionary.get(IMPORTED_REDACTION_INDICATOR)))
.build(),
Type.builder()
.id(DOSSIER_REDACTIONS_TYPE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.entries(toDictionaryEntry(dossierDictionary.get(DOSSIER_REDACTIONS_INDICATOR)))
.build(),
Type.builder()
.id(DOSSIER_AUTHOR_TYPE_ID)
.type(DICTIONARY_AUTHOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe184")
.isHint(hintTypeMap.get(DICTIONARY_AUTHOR))
.isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR))
.isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR))
.rank(rankTypeMap.get(DICTIONARY_AUTHOR))
.entries(toDictionaryEntry(dossierDictionary.get(DICTIONARY_AUTHOR)))
.build(),
Type.builder()
.id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
.type(PUBLISHED_INFORMATION_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe180")
.isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.entries(toDictionaryEntry(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR)))
.build());
}

29
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java
View File

@ -1,8 +1,9 @@
package com.iqser.red.service.redaction.v1.server;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
@ -100,19 +101,23 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest {
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR
+ ":"
+ TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);

9
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -15,7 +15,6 @@ import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.kie.api.runtime.KieContainer;
import org.mockito.Mock;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@ -130,7 +129,7 @@ public class DictionaryServiceTest {
.map(t -> new DictionaryEntry(1, t, 2L, false, "type1"))
.collect(Collectors.toList()));
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1, type2));
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), any(), anyBoolean())).thenReturn(List.of(type1, type2));
when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(type1);
when(dictionaryClient.getDictionaryForType("type2", null)).thenReturn(type2);
when(dictionaryClient.getDictionaryForType("type1", 0L)).thenReturn(type1);
@ -140,7 +139,7 @@ public class DictionaryServiceTest {
var increments = dictionaryService.getDictionaryIncrements("dossierTemplateId", DictionaryVersion.builder().dossierTemplateVersion(0L).build(), null);
when(dictionaryClient.getVersion(any())).thenReturn(1L);
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(type1Updated));
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), any(), anyBoolean())).thenReturn(List.of(type1Updated));
when(dictionaryClient.getDictionaryForType("type1", 1L)).thenReturn(type1Updated);
dictionaryService.updateDictionary("dossierTemplateId", null);
@ -228,8 +227,8 @@ public class DictionaryServiceTest {
assertThat(dossierEntries.size()).isEqualTo(3);
dossierType.setEntries(dossierEntries);
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(List.of(dtType));
when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(List.of(dossierType));
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), any(), anyBoolean())).thenReturn(List.of(dtType));
when(dictionaryClient.getAllTypesForDossier(anyString(), any(), anyBoolean())).thenReturn(List.of(dossierType));
when(dictionaryClient.getDictionaryForType("type1", null)).thenReturn(dtType);
when(dictionaryClient.getDictionaryForType("dossierType", null)).thenReturn(dossierType);

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -219,10 +220,10 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.awt.geom.Rectangle2D;
@ -83,19 +84,10 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
loadDictionaryForTest();
loadTypeForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier());
mockDictionaryCalls(null);

38
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.io.FileOutputStream;
@ -98,42 +99,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_TYPE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build(),
Type.builder()
.id(DOSSIER_AUTHOR_TYPE_ID)
.type(DICTIONARY_AUTHOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe184")
.isHint(hintTypeMap.get(DICTIONARY_AUTHOR))
.isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR))
.isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR))
.rank(rankTypeMap.get(DICTIONARY_AUTHOR))
.build(),
Type.builder()
.id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
.type(PUBLISHED_INFORMATION_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.dossierId(TEST_DOSSIER_ID)
.hexColor("#ffe180")
.isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR))
.isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier());
mockDictionaryCalls(null);
@ -264,6 +233,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse();
}
@Test
public void testPublishedInformationRemovalAtDossierLevel() throws IOException {

16
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -3,7 +3,7 @@ package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.when;
@ -118,19 +118,9 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(getTypeResponseForDossier());
mockDictionaryCalls(null);
@ -400,7 +390,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
System.out.println("noExceptionShouldBeThrownForAnyFiles");
URL url = Thread.currentThread().getContextClassLoader().getResource("files");
Path path = Paths.get(URI.create(url.toString()));
when(dictionaryClient.getDictionaryForType(anyString(), anyLong())).thenReturn(new Type());
when(dictionaryClient.getDictionaryForType(anyString(), any())).thenReturn(new Type());
Files.walk(path)//
.filter(currentPath -> currentPath.toString().endsWith(".pdf"))//
.map(currentPath -> path.getParent().relativize(currentPath))//

27
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.util.List;
@ -74,19 +75,23 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR
+ ":"
+ TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);

23
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server;
import static java.util.Map.entry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
@ -272,19 +273,19 @@ public class RulesTest {
loadDictionaryForTest();
loadTypeForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, anyLong(), false)).thenReturn(getTypeResponse());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, anyLong(), false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
.build()));
mockDictionaryCalls(null);
mockDictionaryCalls(0L);

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import java.awt.Color;
@ -109,10 +110,10 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)

36
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.manualchanges;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import static org.wildfly.common.Assert.assertTrue;
@ -122,20 +123,24 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), false)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, any(), true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, any(), false)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR
+ ":"
+ TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);
@ -240,7 +245,12 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
String manualAddId = UUID.randomUUID().toString();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").user("user").requestDate(OffsetDateTime.now()).build()));
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.user("user")
.requestDate(OffsetDateTime.now())
.build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.realdata;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
@ -159,8 +160,8 @@ public class LiveDataIntegrationTest {
t.setVersion(0L);
});
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), anyBoolean())).thenReturn(types);
when(dictionaryClient.getAllTypesForDossier(anyString(), anyBoolean())).thenReturn(new ArrayList<>());
when(dictionaryClient.getAllTypesForDossierTemplate(anyString(), any(), anyBoolean())).thenReturn(types);
when(dictionaryClient.getAllTypesForDossier(anyString(), any(), anyBoolean())).thenReturn(new ArrayList<>());
when(dictionaryClient.getColors(anyString())).thenReturn(objectMapper.readValue(new ClassPathResource(BASE_DIR + EFSA_SANITISATION_GFL_V1 + "colors.json").getInputStream(),
Colors.class));

25
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@ -98,19 +99,21 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
loadTypeForTest();
loadNerForTest();
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, null, true)).thenReturn(getTypeResponseForTemplate());
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, null, true)).thenReturn(List.of(Type.builder()
.id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
.type(DOSSIER_REDACTIONS_INDICATOR)
.dossierTemplateId(TEST_DOSSIER_ID)
.hexColor("#ffe187")
.isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.isCaseInsensitive(caseInSensitiveMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.isRecommendation(recommendationTypeMap.get(
DOSSIER_REDACTIONS_INDICATOR))
.rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
.build()));
mockDictionaryCalls(null);