From 8eef6ab3faa92da5f4d510131805614abda14c9a Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Thu, 14 Jul 2022 11:50:51 +0200 Subject: [PATCH] RED-4548: PORT: Create SIMPLIFIED_TEXT.json from TEXT.json (for NER-service) --- .../redaction-service-api-v1/pom.xml | 2 +- .../model/SimplifiedSectionText.java | 18 +++++++++++++++++ .../classification/model/SimplifiedText.java | 20 +++++++++++++++++++ .../redaction/service/AnalyzeService.java | 19 +++++++++++++++++- 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 7cfe7a06..b00a497b 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -12,7 +12,7 @@ redaction-service-api-v1 - 1.143.0 + 1.156.20 diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java new file mode 100644 index 00000000..3a015e41 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedSectionText.java @@ -0,0 +1,18 @@ +package com.iqser.red.service.redaction.v1.server.classification.model; + + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimplifiedSectionText { + + private int sectionNumber; + private String text; + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java new file mode 100644 index 00000000..79e8fa66 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SimplifiedText.java @@ -0,0 +1,20 @@ +package com.iqser.red.service.redaction.v1.server.classification.model; + +import java.util.ArrayList; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SimplifiedText { + + private int numberOfPages; + private List simplifiedSectionTexts = new ArrayList<>(); + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 70059b16..da617492 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -2,10 +2,12 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.model.SectionGrid; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; +import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedSectionText; +import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; @@ -104,8 +108,9 @@ public class AnalyzeService { .map(SectionArea::getPage) .collect(Collectors.toSet()), sectionText.getSectionAreas()))); - log.info("Store text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); return AnalyzeResult.builder() @@ -310,4 +315,16 @@ public class AnalyzeService { } } + private SimplifiedText convert(Text text) { + + List sectionTexts = new ArrayList<>(); + int numberOfPages = 0; + + if (text != null) { + numberOfPages = text.getNumberOfPages(); + text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText()))); + } + return SimplifiedText.builder().numberOfPages(numberOfPages).simplifiedSectionTexts(sectionTexts).build(); + } + }