From 116863a0697ae662998f47be784a98d721ee8698 Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Fri, 28 Jun 2024 14:34:04 +0300 Subject: [PATCH 1/2] RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis - update the filtering of the ner entities by super section ids against the sections to be analyzed - unit test added --- .../v1/server/service/AnalyzeService.java | 9 +- .../service/document/NerEntitiesAdapter.java | 2 +- .../AbstractRedactionIntegrationTest.java | 10 ++- .../v1/server/RedactionAcceptanceTest.java | 89 +++++++++++++++++++ ...SA_sanitisation_GFL_v1 3.NER_ENTITIES.json | 1 + 5 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index 0f5057c8..42085162 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -383,7 +383,7 @@ public class AnalyzeService { return new NerEntitiesModel(nerEntitiesModel.getData().entrySet() .stream() // - .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) // + .filter(entry -> sectionsToReanalyseIds.contains(getSuperSectionID(entry.getKey()))) // .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } @@ -399,4 +399,11 @@ public class AnalyzeService { return nerEntities; } + + private static Integer getSuperSectionID(String section) { + + return NerEntitiesAdapter.sectionNumberToTreeId(section) + .get(0); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java index bca1c3ba..37eb8432 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java @@ -177,7 +177,7 @@ public class NerEntitiesAdapter { } - private static List sectionNumberToTreeId(String sectionNumber) { + public static List sectionNumberToTreeId(String sectionNumber) { return Arrays.stream(sectionNumber.split("\\.")) .map(Integer::parseInt) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 00407404..6897306a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -115,13 +115,14 @@ public abstract class AbstractRedactionIntegrationTest { public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; - public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID+ ":" + TEST_DOSSIER_ID; + public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID + ":" + TEST_DOSSIER_ID; public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID; + public static final String DOSSIER_AUTHOR_TYPE_ID = AUTHOR_TYPE_ID + ":" + TEST_DOSSIER_ID; public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID; public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID; @@ -254,8 +255,10 @@ public abstract class AbstractRedactionIntegrationTest { true)); when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR, true)); - when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR, - true)); + when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse( + PUBLISHED_INFORMATION_INDICATOR, + true)); + when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true)); } @@ -354,6 +357,7 @@ public abstract class AbstractRedactionIntegrationTest { .collect(Collectors.toSet())); dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>()); dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>()); + dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>()); falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt") diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 66c71571..314b9c26 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -10,6 +10,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; @@ -25,24 +26,30 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; +import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit.jupiter.SpringExtension; import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; @@ -50,6 +57,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.tenantcommons.TenantContext; +import lombok.SneakyThrows; + @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) @@ -103,6 +112,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) .build(), + Type.builder() + .id(DOSSIER_AUTHOR_TYPE_ID) + .type(DICTIONARY_AUTHOR) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .hexColor("#ffe184") + .isHint(hintTypeMap.get(DICTIONARY_AUTHOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR)) + .isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR)) + .rank(rankTypeMap.get(DICTIONARY_AUTHOR)) + .build(), Type.builder() .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID) .type(PUBLISHED_INFORMATION_INDICATOR) @@ -365,6 +385,75 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { } + @Test + @SneakyThrows + public void testNerEntitiesAfterReanalysis() { + + String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); + + ClassPathResource responseJson = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json"); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), + responseJson.getInputStream()); + + String pdfFile = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf"; + + AnalyzeRequest request = uploadFileToStorage(pdfFile); + + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + request.setAnalysisNumber(1); + dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>()); + mockDictionaryCalls(0L); + + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + String nerValue = "Osip S."; + var nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst() + .orElseThrow(); + assertThat(nerEntity.getEngines().contains(Engine.NER)); + + String dictionaryAddValue = "cooperation"; + ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder() + .value(dictionaryAddValue) + .type(DICTIONARY_AUTHOR) + .user("user") + .addToDossierDictionary(true) + .positions(List.of(Rectangle.builder().topLeftX(180.748f).topLeftY(546.564f).width(56.592f).height(15.408f).page(1).build())) + .type("dossier_redaction") + .fileId(TEST_FILE_ID) + .requestDate(OffsetDateTime.now()) + .annotationId(UUID.randomUUID().toString()) + .build(); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build()); + + request.setAnalysisNumber(2); + dossierDictionary.get(DICTIONARY_AUTHOR).add(dictionaryAddValue); + reanlysisVersions.put(dictionaryAddValue, 2L); + when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L); + mockDictionaryCalls(1L); + + AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request); + + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + EntityLogEntry entityLogEntryAdded = entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getValue().equals(dictionaryAddValue)) + .findFirst() + .get(); + assertEquals(entityLogEntryAdded.getState(), EntryState.APPLIED); + + nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst() + .orElseThrow(); + assertThat(nerEntity.getEngines().contains(Engine.NER)); + dossierDictionary.get(DICTIONARY_AUTHOR).remove(dictionaryAddValue); + + } + + private static IdRemoval buildIdRemoval(String id) { return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json new file mode 100644 index 00000000..2ec7b734 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json @@ -0,0 +1 @@ +{"dossierId": "e2e9bbe6-4ecb-49ea-b278-408e422d742e", "fileId": "6eb06e5f9c23cebb73bf91f69b8cd306", "targetFileExtension": "SIMPLIFIED_TEXT.json.gz", "responseFileExtension": "NER_ENTITIES.json.gz", "X-TENANT-ID": "redaction", "data": {"1": [{"value": "Mustermann Ranya Eikenboom Charalampos Schenk Authors", "startOffset": 140, "endOffset": 193, "type": "CBI_author"}, {"value": "Kara Hunt", "startOffset": 194, "endOffset": 203, "type": "CBI_author"}, {"value": "John", "startOffset": 437, "endOffset": 441, "type": "CBI_author"}], "1.1": [{"value": "Mustermann Ranya Eikenboom Charalampos Schenk Authors", "startOffset": 110, "endOffset": 163, "type": "CBI_author"}, {"value": "Kara Hunt", "startOffset": 164, "endOffset": 173, "type": "CBI_author"}], "1.2": [{"value": "John", "startOffset": 98, "endOffset": 102, "type": "CBI_author"}], "2": [{"value": "Institut Industries", "startOffset": 184, "endOffset": 203, "type": "ORG"}, {"value": "France", "startOffset": 241, "endOffset": 247, "type": "COUNTRY"}, {"value": "Chem", "startOffset": 256, "endOffset": 260, "type": "ORG"}, {"value": "France", "startOffset": 295, "endOffset": 301, "type": "COUNTRY"}, {"value": "Lesdo Industries", "startOffset": 306, "endOffset": 322, "type": "ORG"}, {"value": "Ch\u00e4ppelistr\u00e4ssli, 6078", "startOffset": 324, "endOffset": 346, "type": "CBI_author"}, {"value": "Lungern", "startOffset": 347, "endOffset": 354, "type": "CITY"}, {"value": "Switzerland", "startOffset": 356, "endOffset": 367, "type": "COUNTRY"}, {"value": "Mannheim", "startOffset": 580, "endOffset": 588, "type": "CITY"}, {"value": "Germany", "startOffset": 590, "endOffset": 597, "type": "COUNTRY"}, {"value": "ESP Research", "startOffset": 694, "endOffset": 706, "type": "CBI_author"}, {"value": "Spain", "startOffset": 750, "endOffset": 755, "type": "COUNTRY"}, {"value": "Lilian Rd,", "startOffset": 816, "endOffset": 826, "type": "CBI_author"}, {"value": "Fordsburg", "startOffset": 827, "endOffset": 836, "type": "CITY"}, {"value": "Johannesburg", "startOffset": 838, "endOffset": 850, "type": "CITY"}, {"value": "R. Nice", "startOffset": 920, "endOffset": 927, "type": "CBI_author"}, {"value": "143", "startOffset": 934, "endOffset": 937, "type": "CARDINAL"}, {"value": "Sao Cristovao", "startOffset": 940, "endOffset": 953, "type": "STREET"}, {"value": "Sao Lu\u00eds", "startOffset": 955, "endOffset": 963, "type": "STREET"}, {"value": "MA", "startOffset": 964, "endOffset": 966, "type": "STATE"}, {"value": "Brazil", "startOffset": 979, "endOffset": 985, "type": "COUNTRY"}, {"value": "Rue Jean Baffier", "startOffset": 208, "endOffset": 224, "type": "CBI_author"}, {"value": "Yongsan-gu", "startOffset": 659, "endOffset": 669, "type": "CBI_author"}, {"value": "Seoul, South Korea", "startOffset": 671, "endOffset": 689, "type": "CBI_author"}], "2.1": [{"value": "Institut Industries", "startOffset": 145, "endOffset": 164, "type": "ORG"}, {"value": "France", "startOffset": 202, "endOffset": 208, "type": "COUNTRY"}, {"value": "Chem", "startOffset": 217, "endOffset": 221, "type": "ORG"}, {"value": "France", "startOffset": 256, "endOffset": 262, "type": "COUNTRY"}, {"value": "Lesdo Industries", "startOffset": 267, "endOffset": 283, "type": "ORG"}, {"value": "Ch\u00e4ppelistr\u00e4ssli, 6078", "startOffset": 285, "endOffset": 307, "type": "CBI_author"}, {"value": "Lungern", "startOffset": 308, "endOffset": 315, "type": "CITY"}, {"value": "Switzerland", "startOffset": 317, "endOffset": 328, "type": "COUNTRY"}, {"value": "Mannheim", "startOffset": 541, "endOffset": 549, "type": "CITY"}, {"value": "Germany", "startOffset": 551, "endOffset": 558, "type": "COUNTRY"}, {"value": "ESP Research", "startOffset": 655, "endOffset": 667, "type": "CBI_author"}, {"value": "Spain", "startOffset": 711, "endOffset": 716, "type": "COUNTRY"}, {"value": "Rue Jean Baffier", "startOffset": 169, "endOffset": 185, "type": "CBI_author"}, {"value": "Yongsan-gu", "startOffset": 620, "endOffset": 630, "type": "CBI_author"}, {"value": "Seoul, South Korea", "startOffset": 632, "endOffset": 650, "type": "CBI_author"}], "2.2": [{"value": "Lilian Rd,", "startOffset": 59, "endOffset": 69, "type": "CBI_author"}, {"value": "Fordsburg", "startOffset": 70, "endOffset": 79, "type": "CITY"}, {"value": "Johannesburg", "startOffset": 81, "endOffset": 93, "type": "CITY"}, {"value": "R. Nice", "startOffset": 163, "endOffset": 170, "type": "CBI_author"}, {"value": "143", "startOffset": 177, "endOffset": 180, "type": "CARDINAL"}, {"value": "Sao Cristovao", "startOffset": 183, "endOffset": 196, "type": "STREET"}, {"value": "Sao Lu\u00eds", "startOffset": 198, "endOffset": 206, "type": "STREET"}, {"value": "MA", "startOffset": 207, "endOffset": 209, "type": "STATE"}, {"value": "Brazil", "startOffset": 222, "endOffset": 228, "type": "COUNTRY"}], "4": [{"value": "Carina Madsen", "startOffset": 227, "endOffset": 240, "type": "CBI_author"}], "5": [{"value": "Main St, Los Angeles, CA", "startOffset": 315, "endOffset": 339, "type": "CBI_author"}, {"value": "USA", "startOffset": 347, "endOffset": 350, "type": "COUNTRY"}, {"value": "Suba", "startOffset": 448, "endOffset": 452, "type": "CITY"}, {"value": "Bonne Nouvelle", "startOffset": 480, "endOffset": 494, "type": "CBI_author"}, {"value": "75010", "startOffset": 496, "endOffset": 501, "type": "POSTAL"}, {"value": "Paris", "startOffset": 502, "endOffset": 507, "type": "CITY"}, {"value": "France", "startOffset": 509, "endOffset": 515, "type": "COUNTRY"}, {"value": "Queen Anne St", "startOffset": 523, "endOffset": 536, "type": "CBI_author"}, {"value": "London", "startOffset": 538, "endOffset": 544, "type": "CITY"}, {"value": "Montevideo, Departamento de Montevideo", "startOffset": 373, "endOffset": 411, "type": "CBI_author"}], "6": [{"value": "Francesco Goodman", "startOffset": 156, "endOffset": 173, "type": "CBI_author"}, {"value": "Lucian Terrell and Shaun Juarez WLI Daily Research \u2022 \u2022 \u2022 Mike Herrera Judith Mosley", "startOffset": 175, "endOffset": 258, "type": "CBI_author"}], "7": [{"value": "Luxembourg", "startOffset": 287, "endOffset": 297, "type": "CBI_author"}, {"value": "Japan", "startOffset": 356, "endOffset": 361, "type": "COUNTRY"}], "7.2": [{"value": "Luxembourg", "startOffset": 114, "endOffset": 124, "type": "CBI_author"}, {"value": "Japan", "startOffset": 183, "endOffset": 188, "type": "COUNTRY"}], "8": [{"value": "Feuer A.", "startOffset": 390, "endOffset": 398, "type": "CBI_author"}, {"value": "Greg, M.", "startOffset": 483, "endOffset": 491, "type": "CBI_author"}, {"value": "Laboratory BR2/2 Michael N.", "startOffset": 187, "endOffset": 214, "type": "CBI_author"}, {"value": "Funnarie B.", "startOffset": 298, "endOffset": 309, "type": "CBI_author"}], "8.2": [{"value": "Feuer A.", "startOffset": 237, "endOffset": 245, "type": "CBI_author"}, {"value": "Greg, M.", "startOffset": 330, "endOffset": 338, "type": "CBI_author"}, {"value": "Laboratory BR2/2 Michael N.", "startOffset": 34, "endOffset": 61, "type": "CBI_author"}, {"value": "Funnarie B.", "startOffset": 145, "endOffset": 156, "type": "CBI_author"}], "9": [{"value": "Header Author", "startOffset": 36, "endOffset": 49, "type": "CBI_author"}, {"value": "CBI Author", "startOffset": 53, "endOffset": 63, "type": "ORG"}, {"value": "Michael, J.", "startOffset": 284, "endOffset": 295, "type": "CBI_author"}, {"value": "Byron, C.", "startOffset": 357, "endOffset": 366, "type": "CBI_author"}, {"value": "Corporation TOI/01 William, B.", "startOffset": 412, "endOffset": 442, "type": "DEPARTMENT"}, {"value": "Wayne, L.", "startOffset": 198, "endOffset": 207, "type": "CBI_author"}], "9.2": [{"value": "Michael, J.", "startOffset": 134, "endOffset": 145, "type": "CBI_author"}, {"value": "Byron, C.", "startOffset": 207, "endOffset": 216, "type": "CBI_author"}, {"value": "Corporation TOI/01 William, B.", "startOffset": 262, "endOffset": 292, "type": "DEPARTMENT"}, {"value": "Wayne, L.", "startOffset": 48, "endOffset": 57, "type": "CBI_author"}], "10": [{"value": "Carter Stein", "startOffset": 229, "endOffset": 241, "type": "CBI_author"}, {"value": "Smith", "startOffset": 267, "endOffset": 272, "type": "CBI_author"}, {"value": "Ross Hamster", "startOffset": 309, "endOffset": 321, "type": "CBI_author"}, {"value": "Morpheus Duvall", "startOffset": 358, "endOffset": 373, "type": "CBI_author"}], "10.2": [{"value": "Carter Stein", "startOffset": 61, "endOffset": 73, "type": "CBI_author"}, {"value": "Smith", "startOffset": 99, "endOffset": 104, "type": "CBI_author"}, {"value": "Ross Hamster", "startOffset": 141, "endOffset": 153, "type": "CBI_author"}, {"value": "Morpheus Duvall", "startOffset": 190, "endOffset": 205, "type": "CBI_author"}], "11": [{"value": "Melanie", "startOffset": 486, "endOffset": 493, "type": "CBI_author"}, {"value": "Class Filtered Belkov", "startOffset": 671, "endOffset": 692, "type": "CBI_author"}, {"value": "Sminko", "startOffset": 749, "endOffset": 755, "type": "CBI_author"}, {"value": "Desiree", "startOffset": 364, "endOffset": 371, "type": "CBI_author"}], "11.1": [{"value": "Melanie", "startOffset": 436, "endOffset": 443, "type": "CBI_author"}, {"value": "Desiree", "startOffset": 314, "endOffset": 321, "type": "CBI_author"}], "11.2": [{"value": "Class Filtered Belkov", "startOffset": 49, "endOffset": 70, "type": "CBI_author"}, {"value": "Sminko", "startOffset": 127, "endOffset": 133, "type": "CBI_author"}], "17": [{"value": "Lab", "startOffset": 37, "endOffset": 40, "type": "ORG"}, {"value": "Corporation LABORATORY PROJECT", "startOffset": 218, "endOffset": 248, "type": "DEPARTMENT"}], "18": [{"value": "Japan", "startOffset": 212, "endOffset": 217, "type": "COUNTRY"}, {"value": "B. Rahim", "startOffset": 259, "endOffset": 267, "type": "CBI_author"}, {"value": "C. J.", "startOffset": 268, "endOffset": 273, "type": "CBI_author"}, {"value": "Xinyi Y.", "startOffset": 281, "endOffset": 289, "type": "CBI_author"}, {"value": "Dr. Sergei Vladimir and Professor Alexia Ashford", "startOffset": 367, "endOffset": 415, "type": "CBI_author"}, {"value": "Christine Henri of Daily", "startOffset": 474, "endOffset": 498, "type": "CBI_author"}, {"value": "Nurullah \u00d6zg\u00fcr", "startOffset": 234, "endOffset": 248, "type": "CBI_author"}], "18.1": [{"value": "Japan", "startOffset": 186, "endOffset": 191, "type": "COUNTRY"}, {"value": "B. Rahim", "startOffset": 233, "endOffset": 241, "type": "CBI_author"}, {"value": "C. J.", "startOffset": 242, "endOffset": 247, "type": "CBI_author"}, {"value": "Xinyi Y.", "startOffset": 255, "endOffset": 263, "type": "CBI_author"}, {"value": "Dr. Sergei Vladimir and Professor Alexia Ashford", "startOffset": 341, "endOffset": 389, "type": "CBI_author"}, {"value": "Christine Henri of Daily", "startOffset": 448, "endOffset": 472, "type": "CBI_author"}, {"value": "Nurullah \u00d6zg\u00fcr", "startOffset": 208, "endOffset": 222, "type": "CBI_author"}], "19": [{"value": "Jiwoo", "startOffset": 381, "endOffset": 386, "type": "COUNTRY"}], "19.2": [{"value": "Jiwoo", "startOffset": 30, "endOffset": 35, "type": "COUNTRY"}], "21": [{"value": "+49 (331) 441 551 9. For further questions, please contact +49 331 441551-10. The phone number of the researchers for this project are +49 331-441 551 11 and +49 (331) 441 551 12. For further questions, please contact +49 331 441551-13.", "startOffset": 217, "endOffset": 453, "type": "PHONE"}], "22": [{"value": "luthor.lex1@mail.com Email:", "startOffset": 553, "endOffset": 580, "type": "CBI_author"}, {"value": "luthor.lex3@mail.com", "startOffset": 610, "endOffset": 630, "type": "MAIL"}, {"value": "luthor.lex4@mail.com", "startOffset": 647, "endOffset": 667, "type": "MAIL"}], "25.2": [{"value": "Yendau District Taiwan 109 Contact Point", "startOffset": 83, "endOffset": 123, "type": "CBI_author"}, {"value": "Michelle Carge Fax number", "startOffset": 208, "endOffset": 233, "type": "CBI_author"}, {"value": "+49 331 441 551 29 Fax: +49 331 441 551 30 No: 993-221 Contact: Steffanie Soja Tel.: +49 331 441 551 31 Tel: +49 331 441 551 32 Telephone number: +49 331 441 551 33 Telephone No: +49 331 441 551 34 Telephone: +49 331 441 551 35 Phone No. +49 331 441 551 36 Phone: +49 331 441 551 37 E-mail: sabine.heldt01@mail.com Email: sabine.heldt02@mail.com e-mail: sabine.heldt03@mail.com E-mail address: sabine.heldt04@mail.com No: Redact between No and F/ax Fax Contact: Redact beween contact and T/el Tel", "startOffset": 235, "endOffset": 731, "type": "PHONE"}, {"value": "Nelman Ozbarn Address", "startOffset": 24, "endOffset": 45, "type": "CBI_author"}], "26.2": [{"value": "Xinyi District", "startOffset": 106, "endOffset": 120, "type": "CBI_author"}, {"value": "Sebastian Shaw Alternative", "startOffset": 182, "endOffset": 208, "type": "CBI_author"}, {"value": "Wilson Fisk Fax number", "startOffset": 255, "endOffset": 277, "type": "CBI_author"}, {"value": "+49 331 441 551 38 Fax: +49 331 441 551 39 No: 993-222 Contact: Sabine Heldt Tel.: +49 331 441 551 40 Tel: +49 331 441 551 41 Telephone number: +49 331 441 551 42 Telephone No: +49 331 441 551 43 Telephone: +49 331 441 551 44 Phone No. +49 331 441 551 45 Phone: +49 331 441 551 46 E-mail: sabine.heldt05@mail.com Email: sabine.heldt06@mail.com e-mail: sabine.heldt07@mail.com E-mail address: sabine.heldt08@mail.com No: Redact between No and F/ax Fax Contact: Redact beween contact and T/el Tel", "startOffset": 279, "endOffset": 773, "type": "PHONE"}, {"value": "Norman Osborn Address", "startOffset": 47, "endOffset": 68, "type": "CBI_author"}], "27": [{"value": "Dr. Alan Miller COMPLETION", "startOffset": 170, "endOffset": 196, "type": "CBI_author"}, {"value": "Dr. Alan Milwer STUDY COMPLETION DATE", "startOffset": 231, "endOffset": 268, "type": "CBI_author"}], "29": [{"value": "Research Director", "startOffset": 92, "endOffset": 109, "type": "JOB_TITEL"}, {"value": "Ivan Musk", "startOffset": 110, "endOffset": 119, "type": "CBI_author"}, {"value": "Leon Musk Manager", "startOffset": 200, "endOffset": 217, "type": "STREET"}, {"value": "David Chubb", "startOffset": 80, "endOffset": 91, "type": "CBI_author"}], "39": [{"value": "Manuel, S.", "startOffset": 49, "endOffset": 59, "type": "CBI_author"}, {"value": "Julian, R.", "startOffset": 169, "endOffset": 179, "type": "CBI_author"}, {"value": "Asya. L.", "startOffset": 181, "endOffset": 189, "type": "CBI_author"}, {"value": "Carina, M.", "startOffset": 195, "endOffset": 205, "type": "CBI_author"}, {"value": "Alexandra, H.", "startOffset": 207, "endOffset": 220, "type": "CBI_author"}, {"value": "Yoshua, R.", "startOffset": 296, "endOffset": 306, "type": "CBI_author"}, {"value": "Ralf, H.", "startOffset": 405, "endOffset": 413, "type": "CBI_author"}, {"value": "Ashley B.", "startOffset": 61, "endOffset": 70, "type": "CBI_author"}, {"value": "Valeriya, D.", "startOffset": 92, "endOffset": 104, "type": "CBI_author"}, {"value": "Max, G.", "startOffset": 110, "endOffset": 117, "type": "CBI_author"}, {"value": "Osip S.", "startOffset": 128, "endOffset": 135, "type": "CBI_author"}, {"value": "Iakovos, G.", "startOffset": 137, "endOffset": 148, "type": "CBI_author"}], "39.1": [{"value": "Manuel, S.", "startOffset": 0, "endOffset": 10, "type": "CBI_author"}, {"value": "Julian, R.", "startOffset": 120, "endOffset": 130, "type": "CBI_author"}, {"value": "Asya. L.", "startOffset": 132, "endOffset": 140, "type": "CBI_author"}, {"value": "Carina, M.", "startOffset": 146, "endOffset": 156, "type": "CBI_author"}, {"value": "Alexandra, H.", "startOffset": 158, "endOffset": 171, "type": "CBI_author"}, {"value": "Ashley B.", "startOffset": 12, "endOffset": 21, "type": "CBI_author"}, {"value": "Valeriya, D.", "startOffset": 43, "endOffset": 55, "type": "CBI_author"}, {"value": "Max, G.", "startOffset": 61, "endOffset": 68, "type": "CBI_author"}, {"value": "Osip S.", "startOffset": 79, "endOffset": 86, "type": "CBI_author"}, {"value": "Iakovos, G.", "startOffset": 88, "endOffset": 99, "type": "CBI_author"}], "39.2": [{"value": "Yoshua, R.", "startOffset": 75, "endOffset": 85, "type": "CBI_author"}, {"value": "Ralf, H.", "startOffset": 184, "endOffset": 192, "type": "CBI_author"}], "40": [{"value": "Hamburg", "startOffset": 155, "endOffset": 162, "type": "STATE"}, {"value": "United States Sto\u00dfberger Ltd Katakawa", "startOffset": 163, "endOffset": 200, "type": "COUNTRY"}, {"value": "United States", "startOffset": 277, "endOffset": 290, "type": "COUNTRY"}], "41.2": [{"value": "Sophia,", "startOffset": 890, "endOffset": 897, "type": "CBI_author"}, {"value": "Daniela, B.", "startOffset": 913, "endOffset": 924, "type": "CBI_author"}]}} \ No newline at end of file -- 2.47.2 From 98f8d31fd6ced208703261009f87d70fee1f5910 Mon Sep 17 00:00:00 2001 From: corinaolariu Date: Fri, 28 Jun 2024 14:49:00 +0300 Subject: [PATCH 2/2] RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis - fix sonar issues --- .../v1/server/RedactionAcceptanceTest.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index 314b9c26..7c37f628 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -178,10 +178,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() .orElseThrow(); - assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:"); var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() .orElseThrow(); - assertThat(asyaLyon1.getSection().startsWith("Paragraph:")); + assertThat(asyaLyon1.getSection()).startsWith("Paragraph:"); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); @@ -232,10 +232,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst() .orElseThrow(); - assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:"); var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst() .orElseThrow(); - assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:")); + assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:"); assertEquals(EntryState.SKIPPED, asyaLyon1.getState()); var idRemoval = buildIdRemoval(publishedInformationEntry1.getId()); @@ -387,7 +387,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { @Test @SneakyThrows - public void testNerEntitiesAfterReanalysis() { + void testNerEntitiesAfterReanalysis() { String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES)); @@ -413,7 +413,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { String nerValue = "Osip S."; var nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst() .orElseThrow(); - assertThat(nerEntity.getEngines().contains(Engine.NER)); + assertThat(nerEntity.getEngines()).contains(Engine.NER); String dictionaryAddValue = "cooperation"; ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder() @@ -444,11 +444,11 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { .filter(entityLogEntry -> entityLogEntry.getValue().equals(dictionaryAddValue)) .findFirst() .get(); - assertEquals(entityLogEntryAdded.getState(), EntryState.APPLIED); + assertEquals(EntryState.APPLIED, entityLogEntryAdded.getState()); nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst() .orElseThrow(); - assertThat(nerEntity.getEngines().contains(Engine.NER)); + assertThat(nerEntity.getEngines()).contains(Engine.NER); dossierDictionary.get(DICTIONARY_AUTHOR).remove(dictionaryAddValue); } -- 2.47.2