From 0dd47111c5294d9a9a0a10e174c2832436983205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Thu, 25 Feb 2021 14:55:53 +0100 Subject: [PATCH] RED-1090: Return text with areas and sectionNumber --- .../redaction/v1/model/AnalyzeResult.java | 1 + .../redaction/v1/model/RedactionLogEntry.java | 3 + .../redaction/v1/model/SectionArea.java | 29 ++++++++ .../redaction/v1/model/SectionText.java | 25 +++++++ .../red/service/redaction/v1/model/Text.java | 17 +++++ .../server/classification/model/Document.java | 3 + .../controller/RedactionController.java | 2 + .../v1/server/redaction/model/Entity.java | 4 +- .../service/EntityRedactionService.java | 72 ++++++++++++++++--- .../service/RedactionLogCreatorService.java | 2 + .../segmentation/SectionsBuilderService.java | 13 +++- .../v1/server/RedactionIntegrationTest.java | 41 +++++++++-- 12 files changed, 191 insertions(+), 21 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionText.java create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Text.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java index f6f762bb..56bf4b6a 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResult.java @@ -14,5 +14,6 @@ public class AnalyzeResult { private int numberOfPages; private RedactionLog redactionLog; private SectionGrid sectionGrid; + private Text text; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index e8f5403b..6a278ded 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -40,4 +40,7 @@ public class RedactionLogEntry { @Builder.Default private List comments = new ArrayList<>(); + private int startOffset; + private int endOffset; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java new file mode 100644 index 00000000..0944bcd0 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java @@ -0,0 +1,29 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor +@NoArgsConstructor +@AllArgsConstructor +public class SectionArea { + + @NonNull + private Point topLeft; + + @NonNull + private float width; + + @NonNull + private float height; + + @NonNull + private int page; + + private String header; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionText.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionText.java new file mode 100644 index 00000000..3c8fa02a --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionText.java @@ -0,0 +1,25 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.util.ArrayList; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SectionText { + + private int sectionNumber; + private String text; + + private boolean isTable; + private String headline; + + private List sectionAreas = new ArrayList<>(); + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Text.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Text.java new file mode 100644 index 00000000..a2bc00e9 --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Text.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.model; + +import java.util.ArrayList; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Text { + + private List sectionTexts = new ArrayList<>(); + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java index 99e00b96..71278d58 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java @@ -8,6 +8,7 @@ import java.util.Map; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.SectionGrid; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.model.SectionText; import lombok.Data; import lombok.NoArgsConstructor; @@ -32,4 +33,6 @@ public class Document { private SectionGrid sectionGrid = new SectionGrid(); private long dictionaryVersion; private long rulesVersion; + + private List sectionText = new ArrayList<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 2631ec79..f3d41ab2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -9,6 +9,7 @@ import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.service.redaction.v1.model.SectionGrid; +import com.iqser.red.service.redaction.v1.model.Text; import com.iqser.red.service.redaction.v1.resources.RedactionResource; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Page; @@ -69,6 +70,7 @@ public class RedactionController implements RedactionResource { .redactionLog(new RedactionLog(classifiedDoc.getRedactionLogEntities(), classifiedDoc.getDictionaryVersion(), classifiedDoc .getRulesVersion(), analyzeRequest.getRuleSetId())) .numberOfPages(classifiedDoc.getPages().size()) + .text(new Text(classifiedDoc.getSectionText())) .build(); } catch (Exception e) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index 5f301081..8c8c8952 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -39,7 +39,7 @@ public class Entity { private String textAfter; - public Entity(String word, String type, boolean redaction, String redactionReason, List positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter) { + public Entity(String word, String type, boolean redaction, String redactionReason, List positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end) { this.word = word; this.type = type; @@ -53,6 +53,8 @@ public class Entity { this.isDictionaryEntry = isDictionaryEntry; this.textBefore = textBefore; this.textAfter = textAfter; + this.start = start; + this.end = end; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index cc232049..c8d235c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -17,6 +17,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; +import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Footer; @@ -24,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText; +import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; @@ -32,6 +34,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.model.SectionText; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -86,7 +89,7 @@ public class EntityRedactionService { .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity - .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter())); + .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity.getStart(), entity.getEnd())); } } @@ -110,29 +113,29 @@ public class EntityRedactionService { List tables = paragraph.getTables(); for (Table table : tables) { if (table.getColCount() == 2) { - sectionSearchableTextPairs.addAll(processTableAsOneText(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); } else { - sectionSearchableTextPairs.addAll(processTablePerRow(table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); } sectionNumber.incrementAndGet(); } - sectionSearchableTextPairs.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph + sectionSearchableTextPairs.add(processText(classifiedDoc, paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph .getHeadline(), manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); sectionNumber.incrementAndGet(); } for (Header header : classifiedDoc.getHeaders()) { - sectionSearchableTextPairs.add(processText(header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.add(processText(classifiedDoc, header.getSearchableText(), header.getTextBlocks(), "Header", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); sectionNumber.incrementAndGet(); } for (Footer footer : classifiedDoc.getFooters()) { - sectionSearchableTextPairs.add(processText(footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.add(processText(classifiedDoc, footer.getSearchableText(), footer.getTextBlocks(), "Footer", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); sectionNumber.incrementAndGet(); } for (UnclassifiedText unclassifiedText : classifiedDoc.getUnclassifiedTexts()) { - sectionSearchableTextPairs.add(processText(unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); + sectionSearchableTextPairs.add(processText(classifiedDoc, unclassifiedText.getSearchableText(), unclassifiedText.getTextBlocks(), "", manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber)); sectionNumber.incrementAndGet(); } @@ -169,7 +172,7 @@ public class EntityRedactionService { } - private List processTablePerRow(Table table, ManualRedactions manualRedactions, + private List processTablePerRow(Document classifiedDoc, Table table, ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber) { @@ -181,10 +184,17 @@ public class EntityRedactionService { Map tabularData = new HashMap<>(); int start = 0; List cellStarts = new ArrayList<>(); + SectionText sectionText = new SectionText(); for (Cell cell : row) { + if (CollectionUtils.isEmpty(cell.getTextBlocks())) { continue; } + + SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell + .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage()); + sectionText.getSectionAreas().add(sectionArea); + addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); int cellStart = start; @@ -196,6 +206,7 @@ public class EntityRedactionService { .replaceAll("\n", "") .replaceAll(" ", "") .replaceAll("-", ""); + sectionArea.setHeader(headerName); tabularData.put(headerName, new CellValue(cell.getTextBlocks(), cellStart)); }); } @@ -206,6 +217,7 @@ public class EntityRedactionService { } cellStarts.add(cellStart); start = start + cell.toString().trim().length() + 1; + } Set rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(rowEntities, searchableRow, dictionary, cellStarts); @@ -225,6 +237,14 @@ public class EntityRedactionService { .dictionary(dictionary) .build(), searchableRow)); + if(!local) { + sectionText.setText(searchableRow.toString()); + sectionText.setHeadline(table.getHeadline()); + sectionText.setSectionNumber(sectionNumber.intValue()); + sectionText.setTable(true); + classifiedDoc.getSectionText().add(sectionText); + } + sectionNumber.incrementAndGet(); } @@ -232,18 +252,26 @@ public class EntityRedactionService { } - private List processTableAsOneText(Table table, ManualRedactions manualRedactions, + private List processTableAsOneText(Document classifiedDoc, Table table, ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber) { List sectionSearchableTextPairs = new ArrayList<>(); SearchableText entireTableText = new SearchableText(); + SectionText sectionText = new SectionText(); for (List row : table.getRows()) { for (Cell cell : row) { if (CollectionUtils.isEmpty(cell.getTextBlocks())) { continue; } + + if(!local) { + SectionArea sectionArea = new SectionArea(new Point((float) cell.getX(), (float) cell.getY()), (float) cell + .getWidth(), (float) cell.getHeight(), cell.getTextBlocks().get(0).getSequences().get(0).getPage()); + sectionText.getSectionAreas().add(sectionArea); + } + for (TextBlock textBlock : cell.getTextBlocks()) { entireTableText.addAll(textBlock.getSequences()); } @@ -251,6 +279,7 @@ public class EntityRedactionService { } } + Set rowEntities = findEntities(entireTableText, table.getHeadline(), sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(rowEntities, entireTableText, dictionary); @@ -268,15 +297,38 @@ public class EntityRedactionService { .dictionary(dictionary) .build(), entireTableText)); + if(!local) { + sectionText.setText(entireTableText.toString()); + sectionText.setHeadline(table.getHeadline()); + sectionText.setSectionNumber(sectionNumber.intValue()); + sectionText.setTable(true); + classifiedDoc.getSectionText().add(sectionText); + } + return sectionSearchableTextPairs; } - private SectionSearchableTextPair processText(SearchableText searchableText, List paragraphTextBlocks, + private SectionSearchableTextPair processText(Document classifiedDoc, SearchableText searchableText, List paragraphTextBlocks, String headline, ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber) { + if(!local) { + SectionText sectionText = new SectionText(); + for (TextBlock paragraphTextBlock : paragraphTextBlocks) { + SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock + .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage()); + sectionText.getSectionAreas().add(sectionArea); + } + + sectionText.setText(searchableText.toString()); + sectionText.setHeadline(headline); + sectionText.setSectionNumber(sectionNumber.intValue()); + sectionText.setTable(false); + classifiedDoc.getSectionText().add(sectionText); + } + addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue()); Set entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(entities, searchableText, dictionary); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 3ff5e131..afea9472 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -307,6 +307,8 @@ public class RedactionLogCreatorService { .isDictionaryEntry(entity.isDictionaryEntry()) .textAfter(entity.getTextAfter()) .textBefore(entity.getTextBefore()) + .startOffset(entity.getStart()) + .endOffset(entity.getEnd()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java index d08f633e..73741a5f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java @@ -84,9 +84,16 @@ public class SectionsBuilderService { chunkWords.add(current); prev = current; } - headers.add(new Header(header)); - footers.add(new Footer(footer)); - unclassifiedTexts.add(new UnclassifiedText(unclassifiedText)); + + if(!header.isEmpty()) { + headers.add(new Header(header)); + } + if(!footer.isEmpty()) { + footers.add(new Footer(footer)); + } + if(!unclassifiedText.isEmpty()) { + unclassifiedTexts.add(new UnclassifiedText(unclassifiedText)); + } } Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index f025dbf1..7de575c6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -40,6 +40,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit4.SpringRunner; +import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.configuration.v1.api.model.Colors; import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse; import com.iqser.red.service.configuration.v1.api.model.RulesResponse; @@ -59,6 +60,7 @@ import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.RedactionRequest; import com.iqser.red.service.redaction.v1.model.RedactionResult; +import com.iqser.red.service.redaction.v1.model.SectionText; import com.iqser.red.service.redaction.v1.model.Status; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; @@ -94,6 +96,9 @@ public class RedactionIntegrationTest { @Autowired private RedactionController redactionController; + @Autowired + private ObjectMapper objectMapper; + @MockBean private RulesClient rulesClient; @@ -408,6 +413,7 @@ public class RedactionIntegrationTest { duplicates.entrySet().forEach(entry -> { assertThat(entry.getValue().size()).isEqualTo(1); }); + } long end = System.currentTimeMillis(); @@ -438,15 +444,9 @@ public class RedactionIntegrationTest { @Test public void redactionTest() throws IOException { -// 49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf -// 182 Fludioxonil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf -// 38 A14325E - EU AIR3 - MCP Section 10 - Ecotoxicological studies on the plant protection product.pdf -// 91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf -// 95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf - System.out.println("redactionTest"); long start = System.currentTimeMillis(); - ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); AnalyzeRequest request = AnalyzeRequest.builder() .ruleSetId(TEST_RULESET_ID) @@ -465,6 +465,33 @@ public class RedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Test.json")) { + fileOutputStream.write(objectMapper.writeValueAsBytes(result.getText())); + } + + int correctFound = 0; + loop: + for (RedactionLogEntry redactionLogEntry : result.getRedactionLog().getRedactionLogEntry()) { + for (SectionText sectionText : result.getText().getSectionTexts()) { + if (redactionLogEntry.getType().equals("image")) { + correctFound++; + continue loop; + } + if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) { + String value = sectionText.getText() + .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()); + if (redactionLogEntry.getValue().equalsIgnoreCase(value)) { + correctFound++; + } else { + throw new RuntimeException("WTF"); + } + } + } + } + assertThat(correctFound).isEqualTo(result.getRedactionLog().getRedactionLogEntry().size()); + + System.out.println("correctFound " + correctFound); + long end = System.currentTimeMillis(); System.out.println("duration: " + (end - start));