From c21ad9155cf4c4fc68e366398554497866c701ce Mon Sep 17 00:00:00 2001 From: deiflaender Date: Wed, 9 Dec 2020 13:14:04 +0100 Subject: [PATCH] Avoid ConcurrentModificationException --- .../v1/server/redaction/model/Section.java | 15 ++++++++----- .../model/SectionSearchableTextPair.java | 13 +++++++++++ .../service/EntityRedactionService.java | 22 ++++++++++++++----- .../resources/dictionaries/false_positive.txt | 10 ++++++++- 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index fd91d244..54a70de2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -307,12 +307,17 @@ public class Section { while (matcher.find()) { String match = matcher.group().trim(); - if (match.length() >= 3) { - if(!dictionaryService.getDictionary(type).getEntries().contains(match) && !dictionaryService.getDictionary(RECOMMENDATION_PREFIX + type).getEntries().contains(match)) { - dictionaryService.addToLocalDictionary(RECOMMENDATION_PREFIX + type, match); - } + if (match.length() >= 3 && !dictionaryService.getDictionary(type) + .getEntries() + .contains(match) && !dictionaryService.getDictionary(RECOMMENDATION_PREFIX + type) + .getEntries() + .contains(match)) { + dictionaryService.addToLocalDictionary(RECOMMENDATION_PREFIX + type, match); String lastname = match.split(" ")[0]; - if(!dictionaryService.getDictionary(type).getEntries().contains(lastname) && !dictionaryService.getDictionary(RECOMMENDATION_PREFIX + type).getEntries().contains(lastname)) { + if (!dictionaryService.getDictionary(type).getEntries().contains(lastname) && !dictionaryService + .getDictionary(RECOMMENDATION_PREFIX + type) + .getEntries() + .contains(lastname)) { dictionaryService.addToLocalDictionary(RECOMMENDATION_PREFIX + type, lastname); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java new file mode 100644 index 00000000..996eb50d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class SectionSearchableTextPair { + + private Section section; + private SearchableText searchableText; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 7622e8dc..0d40d4c5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -24,6 +24,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; @@ -90,12 +91,14 @@ public class EntityRedactionService { Set documentEntities = new HashSet<>(); int sectionNumber = 1; + List sectionSearchableTextPairs = new ArrayList<>(); for (Paragraph paragraph : classifiedDoc.getParagraphs()) { SearchableText searchableText = paragraph.getSearchableText(); List tables = paragraph.getTables(); + for (Table table : tables) { boolean singleCellTable = table.getRowCount() == 1 && table.getColCount() == 1; for (List row : table.getRows()) { @@ -124,7 +127,7 @@ public class EntityRedactionService { } Set rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber, localEntries); - Section analysedRowSection = droolsExecutionService.executeRules(Section.builder() + sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() .dictionaryService(dictionaryService) .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream .concat(rowEntities.stream(), hintsPerSectionNumber.get(sectionNumber).stream()) @@ -134,9 +137,8 @@ public class EntityRedactionService { .headline(table.getHeadline()) .sectionNumber(sectionNumber) .tabularData(tabularData) - .build()); + .build(), searchableRow)); - documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), searchableRow)); sectionNumber++; } sectionNumber++; @@ -144,7 +146,8 @@ public class EntityRedactionService { addSectionToManualRedactions(paragraph.getTextBlocks(), manualRedactions, paragraph.getHeadline(), sectionNumber); Set entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber, localEntries); - Section analysedSection = droolsExecutionService.executeRules(Section.builder() + + sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() .dictionaryService(dictionaryService) .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(sectionNumber) ? Stream .concat(entities.stream(), hintsPerSectionNumber.get(sectionNumber).stream()) @@ -153,11 +156,18 @@ public class EntityRedactionService { .searchText(searchableText.toString()) .headline(paragraph.getHeadline()) .sectionNumber(sectionNumber) - .build()); + .build(), searchableText)); - documentEntities.addAll(clearAndFindPositions(analysedSection.getEntities(), searchableText)); sectionNumber++; } + + + sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { + Section analysedRowSection = droolsExecutionService.executeRules(sectionSearchableTextPair.getSection()); + documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), sectionSearchableTextPair + .getSearchableText())); + }); + return documentEntities; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt index 95da0f83..25e3e9e9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/false_positive.txt @@ -1,2 +1,10 @@ Long-term -Brown liquid \ No newline at end of file +Brown liquid +Brown solid +Hand-held +Manual-Hand held +Manual-Hand held +Weight: +Sprague +Weight and length +Aeration: Gentle \ No newline at end of file