diff --git a/redaction-service-v1/redaction-service-server-v1/pom.xml b/redaction-service-v1/redaction-service-server-v1/pom.xml index 2cec099b..82fa4b39 100644 --- a/redaction-service-v1/redaction-service-server-v1/pom.xml +++ b/redaction-service-v1/redaction-service-server-v1/pom.xml @@ -20,7 +20,7 @@ com.iqser.red.service configuration-service-api-v1 - 1.0.12 + 1.1.7 org.drools diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index ef023c65..7966d0b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.awt.Color; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -11,6 +12,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; +import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.TypeResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResult; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; @@ -51,6 +53,9 @@ public class DictionaryService { @Getter private float[] requestRemoveColor; + @Getter + private float[] notRedactedColor; + public void updateDictionary() { @@ -69,7 +74,7 @@ public class DictionaryService { if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) { entryColors = typeResponse.getTypes() .stream() - .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor)); + .collect(Collectors.toMap(TypeResult::getType, t -> convertColor(t.getHexColor()))); hintTypes = typeResponse.getTypes() .stream() .filter(TypeResult::isHint) @@ -80,12 +85,16 @@ public class DictionaryService { .filter(TypeResult::isCaseInsensitive) .map(TypeResult::getType) .collect(Collectors.toList()); - dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, this::convertEntries)); - defaultColor = dictionaryClient.getDefaultColor().getColor(); + dictionary = entryColors.keySet() + .stream() + .collect(Collectors.toMap(type -> type, this::convertEntries)); - // TODO get colors from configuration service. - requestAddColor = new float[]{0f, 1f, 0.8f}; - requestRemoveColor = new float[]{0f, 1f, 0.8f}; + + Colors colors = dictionaryClient.getColors(); + defaultColor = convertColor(colors.getDefaultColor()); + requestAddColor = convertColor(colors.getRequestAdd()); + requestRemoveColor = convertColor(colors.getRequestRemove()); + notRedactedColor = convertColor(colors.getNotRedacted()); } } catch (FeignException e) { log.warn("Got some unknown feignException", e); @@ -95,6 +104,7 @@ public class DictionaryService { private Set convertEntries(String s) { + if (caseInsensitiveTypes.contains(s)) { return dictionaryClient.getDictionaryForType(s) .getEntries() @@ -105,4 +115,11 @@ public class DictionaryService { return new HashSet<>(dictionaryClient.getDictionaryForType(s).getEntries()); } + + private float[] convertColor(String hex) { + + Color color = Color.decode(hex); + return new float[]{color.getRed() / 255f, color.getGreen() / 255f, color.getBlue() / 255f}; + } + } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java index d02b5fe2..e5741ee3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java @@ -396,7 +396,7 @@ public class AnnotationHighlightService { return dictionaryService.getRequestRemoveColor(); } if (!entity.isRedaction() && !isHint(entity)) { - return new float[]{0.627f, 0.627f, 0.627f}; + return dictionaryService.getNotRedactedColor(); } if (!dictionaryService.getEntryColors().containsKey(entity.getType())) { return dictionaryService.getDefaultColor(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index abdbc461..0a0f2bc3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -39,7 +39,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; -import com.iqser.red.service.configuration.v1.api.model.DefaultColor; +import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse; import com.iqser.red.service.configuration.v1.api.model.RulesResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResponse; @@ -82,9 +82,10 @@ public class RedactionIntegrationTest { private DictionaryClient dictionaryClient; private final Map> dictionary = new HashMap<>(); - private final Map typeColorMap = new HashMap<>(); + private final Map typeColorMap = new HashMap<>(); private final Map hintTypeMap = new HashMap<>(); private final Map caseInSensitiveMap = new HashMap<>(); + private final Colors colors = new Colors(); @TestConfiguration public static class RedactionIntegrationTestConfiguration { @@ -96,8 +97,8 @@ public class RedactionIntegrationTest { KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8)); - kieFileSystem.write("src/test/resources/drools/rules.drl", - kieServices.getResources().newInputStreamResource(input)); + kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources() + .newInputStreamResource(input)); KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); kieBuilder.buildAll(); KieModule kieModule = kieBuilder.getKieModule(); @@ -122,13 +123,11 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE)); when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE)); when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR)); - when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse( - NO_REDACTION_INDICATOR)); - when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse( - REDACTION_INDICATOR)); + when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR)); when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY)); when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT)); - when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f})); + when(dictionaryClient.getColors()).thenReturn(colors); } @@ -136,44 +135,44 @@ public class RedactionIntegrationTest { dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/names.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/sponsor_companies.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/vertebrates.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/addresses.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/hint_only.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/must_redact.txt") - .stream() - .map(this::cleanDictionaryEntry) - .collect(Collectors.toSet())); + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); } @@ -185,14 +184,14 @@ public class RedactionIntegrationTest { private void loadTypeForTest() { - typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0}); - typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1}); - typeColorMap.put(NAME_CODE, new float[]{1, 1, 0}); - typeColorMap.put(SPONSOR, new float[]{.5f, .5f, .5f}); - typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f}); - typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f}); - typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f}); - typeColorMap.put(MUST_REDACT, new float[]{1, 0, 0}); + typeColorMap.put(VERTEBRATES_CODE, "#00ff00"); + typeColorMap.put(ADDRESS_CODE, "#00ffff"); + typeColorMap.put(NAME_CODE, "#ffff00"); + typeColorMap.put(SPONSOR, "#2c21fc"); + typeColorMap.put(NO_REDACTION_INDICATOR, "#e600ff"); + typeColorMap.put(REDACTION_INDICATOR, "#ff7700"); + typeColorMap.put(HINT_ONLY, "#00fcb1"); + typeColorMap.put(MUST_REDACT, "#ff0000"); hintTypeMap.put(VERTEBRATES_CODE, true); hintTypeMap.put(ADDRESS_CODE, false); @@ -211,6 +210,11 @@ public class RedactionIntegrationTest { caseInSensitiveMap.put(REDACTION_INDICATOR, true); caseInSensitiveMap.put(HINT_ONLY, true); caseInSensitiveMap.put(MUST_REDACT, true); + + colors.setDefaultColor("#acfc00"); + colors.setNotRedacted("#cccccc"); + colors.setRequestAdd("#04b093"); + colors.setRequestRemove("#04b093"); } @@ -220,7 +224,7 @@ public class RedactionIntegrationTest { .stream() .map(typeColor -> TypeResult.builder() .type(typeColor.getKey()) - .color(typeColor.getValue()) + .hexColor(typeColor.getValue()) .isHint(hintTypeMap.get(typeColor.getKey())) .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) .build()) @@ -232,7 +236,7 @@ public class RedactionIntegrationTest { private DictionaryResponse getDictionaryResponse(String type) { return DictionaryResponse.builder() - .color(typeColorMap.get(type)) + .hexColor(typeColorMap.get(type)) .entries(dictionary.get(type)) .isHint(hintTypeMap.get(type)) .isCaseInsensitive(caseInSensitiveMap.get(type)) @@ -285,8 +289,7 @@ public class RedactionIntegrationTest { System.out.println("redactionTest"); long start = System.currentTimeMillis(); - ClassPathResource pdfFileResource = new ClassPathResource( - "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -310,8 +313,7 @@ public class RedactionIntegrationTest { System.out.println("testTableRedaction"); long start = System.currentTimeMillis(); - ClassPathResource pdfFileResource = new ClassPathResource( - "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -338,22 +340,22 @@ public class RedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); - Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build(); - manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().id("0836727c3508a0b2ea271da69c04cc2f").approved(false).build())); + Comment comment = Comment.builder() + .date(OffsetDateTime.now()) + .user("TEST_USER") + .text("This is a comment test") + .build(); + manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() + .id("0836727c3508a0b2ea271da69c04cc2f") + .approved(false) + .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); manualRedactionEntry.setComments(List.of(comment)); manualRedactionEntry.setType("name"); manualRedactionEntry.setValue("O'Loughlin C.K."); manualRedactionEntry.setReason("Manual Redaction"); - manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), - 7.648041f, - 43.72262f, - 1), - new Rectangle(new Point(384.83517f, 241.282f), - 7.648041f, - 17.043358f, - 1))); + manualRedactionEntry.setPositions(List.of(new Rectangle(new Point(375.61096f, 241.282f), 7.648041f, 43.72262f, 1), new Rectangle(new Point(384.83517f, 241.282f), 7.648041f, 17.043358f, 1))); manualRedactions.getEntriesToAdd().add(manualRedactionEntry); @@ -378,8 +380,7 @@ public class RedactionIntegrationTest { public void classificationTest() throws IOException { System.out.println("classificationTest"); - ClassPathResource pdfFileResource = new ClassPathResource( - "files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -397,8 +398,7 @@ public class RedactionIntegrationTest { public void sectionsTest() throws IOException { System.out.println("sectionsTest"); - ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " + - "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " + "Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -416,8 +416,7 @@ public class RedactionIntegrationTest { public void htmlTablesTest() throws IOException { System.out.println("htmlTablesTest"); - ClassPathResource pdfFileResource = new ClassPathResource( - "files/Minimal Examples/line_breaks.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/line_breaks.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -435,8 +434,7 @@ public class RedactionIntegrationTest { public void htmlTableRotationTest() throws IOException { System.out.println("htmlTableRotationTest"); - ClassPathResource pdfFileResource = new ClassPathResource( - "files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); RedactionRequest request = RedactionRequest.builder() .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) @@ -499,8 +497,7 @@ public class RedactionIntegrationTest { if (resource == null) { throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); } - try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), - StandardCharsets.UTF_8))) { + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { StringBuilder sb = new StringBuilder(); String str; while ((str = br.readLine()) != null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index 77116f34..aaa2a96b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -35,7 +35,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; -import com.iqser.red.service.configuration.v1.api.model.DefaultColor; +import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse; import com.iqser.red.service.configuration.v1.api.model.RulesResponse; import com.iqser.red.service.configuration.v1.api.model.TypeResponse; @@ -380,12 +380,19 @@ public class EntityRedactionServiceTest { when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules)); TypeResponse typeResponse = TypeResponse.builder() .types(Arrays.asList( - TypeResult.builder().type(NAME_CODE).color(new float[]{1, 1, 0}).build(), - TypeResult.builder().type(ADDRESS_CODE).color(new float[]{0, 1, 1}).build())) + TypeResult.builder().type(NAME_CODE).hexColor("#ffff00").build(), + TypeResult.builder().type(ADDRESS_CODE).hexColor("#00ffff").build())) .build(); when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet()); when(dictionaryClient.getAllTypes()).thenReturn(typeResponse); - when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor()); + + Colors colors = new Colors(); + colors.setDefaultColor("#acfc00"); + colors.setNotRedacted("#cccccc"); + colors.setRequestAdd("#04b093"); + colors.setRequestRemove("#04b093"); + + when(dictionaryClient.getColors()).thenReturn(colors); }