diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java index d59c4215..cbee642b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java @@ -1,16 +1,22 @@ package com.iqser.red.service.redaction.v1.server.parsing.model; +import org.apache.pdfbox.text.TextPosition; +import org.springframework.beans.BeanUtils; + import com.dslplatform.json.CompiledJson; import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; + +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.SneakyThrows; -import org.apache.pdfbox.text.TextPosition; -import org.springframework.beans.BeanUtils; @Data +@Builder @NoArgsConstructor +@AllArgsConstructor @CompiledJson public class RedTextPosition { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java index 1968f63e..704b383b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java @@ -1,24 +1,30 @@ package com.iqser.red.service.redaction.v1.server.parsing.model; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.pdfbox.text.TextPosition; + import com.dslplatform.json.CompiledJson; import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; + +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.pdfbox.text.TextPosition; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; @Slf4j @Data +@Builder @CompiledJson @NoArgsConstructor +@AllArgsConstructor @JsonIgnoreProperties({"empty"}) public class TextPositionSequence implements CharSequence { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index a08e2de5..47b09af1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -1,17 +1,23 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - -import lombok.Data; -import lombok.EqualsAndHashCode; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor @EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Entity implements ReasonHolder { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java index 6784707d..5e125a8d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java @@ -1,15 +1,18 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; - import java.util.ArrayList; import java.util.List; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; + @Data +@Builder @RequiredArgsConstructor @AllArgsConstructor @EqualsAndHashCode diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java index 182658a5..ebc17827 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java @@ -1,14 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; - import com.dslplatform.json.CompiledJson; import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; + import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; @Data +@Builder @CompiledJson @NoArgsConstructor @AllArgsConstructor diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 1042de35..815d1a2f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1,28 +1,37 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; - -import com.iqser.red.service.redaction.v1.model.ArgumentType; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.redaction.v1.model.FileAttribute; -import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; -import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; -import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; -import lombok.Builder; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + +import com.iqser.red.service.redaction.v1.model.ArgumentType; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.model.SectionArea; +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; +import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; +import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + @Data @Slf4j @Builder @@ -61,6 +70,9 @@ public class Section { @Builder.Default private List fileAttributes = new ArrayList<>(); + @Builder.Default + private List sectionAreas = new ArrayList<>(); + @SuppressWarnings("unused") @WhenCondition @@ -257,10 +269,10 @@ public class Section { return tabularData != null && tabularData.containsKey(cleanHeaderName) && tabularData.get(cleanHeaderName).toString().equals(value); } - @SuppressWarnings("unused") + @ThenAction - public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { + @SuppressWarnings("unused") + public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String prefixPattern, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.INTEGER) int group) { expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null); } @@ -693,6 +705,38 @@ public class Section { } + @ThenAction + @SuppressWarnings("unused") + public void redactSection(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + + for (SectionArea sectionArea : sectionAreas) { + + RedRectangle2D position = RedRectangle2D.builder() + .height(sectionArea.getHeight()) + .width(sectionArea.getWidth()) + .x(sectionArea.getTopLeft().getX()) + .y(sectionArea.getTopLeft().getY()) + .build(); + + Image image = Image.builder() + .page(sectionArea.getPage()) + .position(position) + .redaction(true) + .hasTransparency(false) + .sectionNumber(sectionNumber) + .section(sectionArea.getHeader()) + .matchedRule(ruleNumber) + .legalBasis(legalBasis) + .redactionReason(reason) + .type(type) + .build(); + + images.add(image); + } + } + + private void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 070b6837..4880661e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,5 +1,18 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.lang3.StringUtils; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; @@ -8,20 +21,23 @@ import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; +import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.Image; +import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.redaction.model.Section; +import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Slf4j @Service @@ -116,8 +132,7 @@ public class EntityRedactionService { .dictionaryTypes(dictionary.getTypes()) .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream .concat(entities.getEntities().stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()) - .stream()) - .collect(Collectors.toSet()) : entities.getEntities()) + .stream()).collect(Collectors.toSet()) : entities.getEntities()) .nerEntities(entities.getNerEntities()) .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks()) .searchText(reanalysisSection.getSearchableText().toString()) @@ -127,6 +142,7 @@ public class EntityRedactionService { .searchableText(reanalysisSection.getSearchableText()) .dictionary(dictionary) .images(reanalysisSection.getImages()) + .sectionAreas(reanalysisSection.getSectionAreas()) .fileAttributes(analyzeRequest.getFileAttributes()) .build(), reanalysisSection.getSearchableText()));