diff --git a/redaction-service-v1/pom.xml b/redaction-service-v1/pom.xml index 5b3aaadd..8b464562 100644 --- a/redaction-service-v1/pom.xml +++ b/redaction-service-v1/pom.xml @@ -23,20 +23,24 @@ 2.0.24 + 1.9.9 - + + com.dslplatform + dsl-json-java8 + ${dsljson.version} + com.iqser.red platform-commons-dependency - 1.11.0 + 1.13.0 import pom - org.apache.pdfbox pdfbox @@ -47,9 +51,7 @@ pdfbox-tools ${pdfbox.version} - - @@ -58,12 +60,10 @@ org.sonarsource.scanner.maven sonar-maven-plugin - 3.9.0.2155 org.owasp dependency-check-maven - 6.3.1 ALL @@ -88,27 +88,5 @@ - - - org.jacoco - jacoco-maven-plugin - 0.8.8 - - - prepare-agent - - prepare-agent - - - - report - - report-aggregate - - verify - - - - diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 9b52d48a..eec6528c 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -16,11 +16,20 @@ + + + + com.dslplatform + dsl-json-java8 + ${dsljson.version} + + org.springframework spring-web true + com.iqser.red.service persistence-service-api-v1 @@ -32,5 +41,21 @@ + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + lombok.launch.AnnotationProcessorHider$AnnotationProcessor + com.dslplatform.json.processor.CompiledJsonAnnotationProcessor + + + + + diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java index 9abd5b66..95c547a5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java @@ -22,7 +22,7 @@ public class ManualChange { private OffsetDateTime processedDate; private OffsetDateTime requestedDate; private String userId; - private Map propertyChanges = new HashMap<>(); + private Map propertyChanges = new HashMap<>(); public boolean isProcessed() { return processedDate != null; @@ -42,7 +42,7 @@ public class ManualChange { return this; } - public ManualChange withChange(String property, Object value) { + public ManualChange withChange(String property, String value) { this.propertyChanges.put(property, value); return this; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java index d6589b7d..384f16b9 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Point.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java index f461b0c7..56d63c3f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 88de3a3a..35a1403c 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -1,14 +1,19 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; +import java.util.ArrayList; import java.util.List; @Data +@CompiledJson @AllArgsConstructor +@NoArgsConstructor public class RedactionLog { @@ -23,8 +28,8 @@ public class RedactionLog { */ private int analysisNumber; - private List redactionLogEntry; - private List legalBasis; + private List redactionLogEntry = new ArrayList<>(); + private List legalBasis = new ArrayList<>(); private long dictionaryVersion = -1; private long dossierDictionaryVersion = -1; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java new file mode 100644 index 00000000..ec24508e --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogComment.java @@ -0,0 +1,22 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.OffsetDateTime; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RedactionLogComment { + + private long id; + private String user; + private String text; + private String annotationId; + private String fileId; + private OffsetDateTime date; + private OffsetDateTime softDeletedTime; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index 74550267..e9c6514f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import lombok.*; @@ -41,7 +42,7 @@ public class RedactionLogEntry { private String textAfter; @Builder.Default - private List comments = new ArrayList<>(); + private List comments = new ArrayList<>(); private int startOffset; private int endOffset; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java new file mode 100644 index 00000000..4290551f --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogLegalBasis.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class RedactionLogLegalBasis { + + private String name; + private String description; + private String reason; + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java index eb71f6fb..bd905750 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionArea.java @@ -1,29 +1,17 @@ package com.iqser.red.service.redaction.v1.model; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; +import com.dslplatform.json.CompiledJson; +import lombok.*; @Data -@RequiredArgsConstructor @NoArgsConstructor @AllArgsConstructor public class SectionArea { - @NonNull private Point topLeft; - - @NonNull private float width; - - @NonNull private float height; - - @NonNull private int page; - private String header; public boolean contains(Rectangle other) { diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java index d1f027a1..818f611f 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -8,6 +9,7 @@ import lombok.RequiredArgsConstructor; import java.util.*; @Data +@CompiledJson @AllArgsConstructor @NoArgsConstructor public class SectionGrid { @@ -17,13 +19,14 @@ public class SectionGrid { private List sections = new ArrayList<>(); @Data - @RequiredArgsConstructor + @NoArgsConstructor + @AllArgsConstructor public static class SectionGridSection { - private final int sectionNumber; - private final String headline; - private final Set pages; - private final List sectionAreas; + private int sectionNumber; + private String headline; + private Set pages; + private List sectionAreas; } } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 3a21bb47..26e1caab 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -11,22 +12,12 @@ import java.util.List; @Data @AllArgsConstructor @NoArgsConstructor -@RequiredArgsConstructor public class SectionRectangle { - @NonNull private Point topLeft; - - @NonNull private float width; - - @NonNull private float height; - - @NonNull private int part; - - @NonNull private int numberOfParts; private List tableCells; diff --git a/redaction-service-v1/redaction-service-server-v1/pom.xml b/redaction-service-v1/redaction-service-server-v1/pom.xml index 0d33acba..2a48c8a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/pom.xml +++ b/redaction-service-v1/redaction-service-server-v1/pom.xml @@ -12,11 +12,12 @@ redaction-service-server-v1 - 7.59.0.Final - 7.59.0.Final - 1.16.1 - 3.0.3 - 1.4.0 + 7.68.0.Final + 7.68.0.Final + 1.18.2 + 3.28.0-GA + 0.6.3 + 2.13.2 @@ -26,15 +27,21 @@ - org.ahocorasick - ahocorasick - 0.6.3 + com.fasterxml.jackson.module + jackson-module-afterburner + ${jackson.version} - org.openjdk.jol - jol-core - 0.10 + org.ahocorasick + ahocorasick + ${ahocorasick.version} + + + + org.javassist + javassist + ${javaassist.version} @@ -62,22 +69,6 @@ guava - - org.apache.pdfbox - jbig2-imageio - ${pdfbox.jbig2-imageio.version} - - - com.github.jai-imageio - jai-imageio-core - ${jai-imageio.version} - - - com.github.jai-imageio - jai-imageio-jpeg2000 - ${jai-imageio.version} - - com.iqser.red.commons @@ -126,6 +117,18 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + + lombok.launch.AnnotationProcessorHider$AnnotationProcessor + com.dslplatform.json.processor.CompiledJsonAnnotationProcessor + + + + pl.project13.maven diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java index b88a16b7..9d1f857d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Footer.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.AllArgsConstructor; @@ -14,6 +15,7 @@ public class Footer { private List textBlocks; @JsonIgnore + @JsonAttribute(ignore = true) public SearchableText getSearchableText() { SearchableText searchableText = new SearchableText(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java index 133e0245..448962ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Header.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.AllArgsConstructor; @@ -14,6 +15,7 @@ public class Header { private List textBlocks; @JsonIgnore + @JsonAttribute(ignore = true) public SearchableText getSearchableText() { SearchableText searchableText = new SearchableText(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java index 77649132..cf6ced53 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; @@ -14,6 +16,7 @@ import java.util.*; @Data @Builder +@CompiledJson @NoArgsConstructor @AllArgsConstructor public class SectionText { @@ -38,6 +41,7 @@ public class SectionText { } @JsonIgnore + @JsonAttribute(ignore = true) public SearchableText getSearchableText() { SearchableText searchableText = new SearchableText(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java index 4df1ace3..cf873c09 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Text.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -8,6 +9,7 @@ import java.util.ArrayList; import java.util.List; @Data +@CompiledJson @NoArgsConstructor @AllArgsConstructor public class Text { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java index 396006ae..6bbb2251 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; @@ -15,6 +17,7 @@ import java.util.List; @AllArgsConstructor @Builder @Data +@CompiledJson @NoArgsConstructor public class TextBlock extends AbstractTextContainer { @@ -121,6 +124,7 @@ public class TextBlock extends AbstractTextContainer { @Override @JsonIgnore + @JsonAttribute(ignore = true) public String getText() { StringBuilder sb = new StringBuilder(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java index 0d51a4f8..079df056 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/UnclassifiedText.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.classification.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import lombok.AllArgsConstructor; @@ -14,6 +15,7 @@ public class UnclassifiedText { private List textBlocks; @JsonIgnore + @JsonAttribute(ignore = true) public SearchableText getSearchableText() { SearchableText searchableText = new SearchableText(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java index b86a1b66..752e13a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecogintionEntity.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.client.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -7,6 +8,7 @@ import lombok.NoArgsConstructor; @Data @Builder +@CompiledJson @AllArgsConstructor @NoArgsConstructor public class EntityRecogintionEntity { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java index 88c6f475..cc628f4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java @@ -4,18 +4,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; @Data -@Builder -@AllArgsConstructor +@CompiledJson @NoArgsConstructor +@AllArgsConstructor public class NerEntities { - @Builder.Default private Map> data = new HashMap<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/memory/MemoryStats.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/memory/MemoryStats.java deleted file mode 100644 index 2491d9eb..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/memory/MemoryStats.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.memory; - -import lombok.extern.slf4j.Slf4j; - -import java.text.CharacterIterator; -import java.text.StringCharacterIterator; - -@Slf4j -public class MemoryStats { - - - public static void printMemoryStats() { - log.info("\n\n ------------------------------ \n" + - " Used Memory: " + humanReadableByteCountBin(getUsedMemory()) + "\n" + - " Free Memory: " + humanReadableByteCountBin(getFreeMemory()) + "\n" + - " Total Memory: " + humanReadableByteCountBin(getTotalMemory()) + "\n" + - " Max Memory: " + humanReadableByteCountBin(getMaxMemory()) + "\n" + - "\n ------------------------------ \n"); - } - - - public static String humanReadableByteCountBin(long bytes) { - long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes); - if (absB < 1024) { - return bytes + " B"; - } - long value = absB; - CharacterIterator ci = new StringCharacterIterator("KMGTPE"); - for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) { - value >>= 10; - ci.next(); - } - value *= Long.signum(bytes); - return String.format("%.1f %ciB", value / 1024.0, ci.current()); - } - - private static long getMaxMemory() { - return Runtime.getRuntime().maxMemory(); - } - - private static long getUsedMemory() { - return getMaxMemory() - getFreeMemory(); - } - - private static long getTotalMemory() { - return Runtime.getRuntime().totalMemory(); - } - - private static long getFreeMemory() { - return Runtime.getRuntime().freeMemory(); - } -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index ba534a83..8746bf71 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -1,52 +1,26 @@ package com.iqser.red.service.redaction.v1.server.parsing; -import java.awt.geom.Point2D; -import java.awt.geom.Rectangle2D; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.pdfbox.contentstream.operator.Operator; -import org.apache.pdfbox.contentstream.operator.OperatorName; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor; -import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor; -import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor; -import org.apache.pdfbox.contentstream.operator.state.SetFlatness; -import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle; -import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern; -import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle; -import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit; -import org.apache.pdfbox.contentstream.operator.state.SetLineWidth; -import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent; -import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize; -import org.apache.pdfbox.cos.COSBase; -import org.apache.pdfbox.cos.COSName; -import org.apache.pdfbox.cos.COSNumber; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.graphics.PDXObject; -import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject; -import org.apache.pdfbox.text.TextPosition; -import org.apache.pdfbox.util.Matrix; - import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; - import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.pdfbox.contentstream.operator.Operator; +import org.apache.pdfbox.contentstream.operator.OperatorName; +import org.apache.pdfbox.contentstream.operator.color.*; +import org.apache.pdfbox.contentstream.operator.state.*; +import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize; +import org.apache.pdfbox.cos.COSBase; +import org.apache.pdfbox.cos.COSNumber; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.text.TextPosition; + +import java.awt.geom.Point2D; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; @Slf4j public class PDFLinesTextStripper extends PDFTextStripper { @@ -66,8 +40,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { private int minCharHeight; @Getter private int maxCharHeight; - @Getter - private List images = new ArrayList<>(); private float path_x; private float path_y; @@ -183,9 +155,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { graphicsPath.clear(); break; -// case OperatorName.DRAW_OBJECT: -// processImageOperation(arguments); -// break; } @@ -193,32 +162,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { } - protected void processImageOperation(List arguments) { - - try { - COSName objectName = (COSName) arguments.get(0); - PDXObject xobject = getResources().getXObject(objectName); - if (xobject instanceof PDImageXObject) { - PDImageXObject image = (PDImageXObject) xobject; - Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix(); - - Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY()); - - // Memory Hack - sofReference kills me - FieldUtils.writeField(image, "cachedImageSubsampling", -1, true); - - if (rect.getHeight() > 2 && rect.getWidth() > 2) { - this.images.add(new PdfImage(image.getImage(), rect, pageNumber, image.getImage() - .getColorModel() - .hasAlpha())); - } - } - } catch (Exception e) { - log.warn("Problem during image extraction: {}", e.getMessage()); - } - } - - private float floatValue(COSBase value) { if (value instanceof COSNumber) { @@ -375,7 +318,6 @@ public class PDFLinesTextStripper extends PDFTextStripper { minCharHeight = Integer.MAX_VALUE; maxCharHeight = 0; textPositionSequences.clear(); - images = new ArrayList<>(); rulings.clear(); graphicsPath.clear(); path_x = 0.0f; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java index f9d9b9e2..d59c4215 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/RedTextPosition.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.parsing.model; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.NoArgsConstructor; @@ -9,6 +11,7 @@ import org.springframework.beans.BeanUtils; @Data @NoArgsConstructor +@CompiledJson public class RedTextPosition { private String textMatrix; @@ -26,14 +29,17 @@ public class RedTextPosition { // not used in reanalysis @JsonIgnore + @JsonAttribute(ignore = true) private float widthOfSpace; // not used in reanalysis @JsonIgnore + @JsonAttribute(ignore = true) private float fontSizeInPt; // not used in reanalysis @JsonIgnore + @JsonAttribute(ignore = true) private String fontName; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java index b8b9f208..1968f63e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java @@ -1,22 +1,23 @@ package com.iqser.red.service.redaction.v1.server.parsing.model; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.pdfbox.text.TextPosition; + import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import org.apache.pdfbox.text.TextPosition; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.iqser.red.service.redaction.v1.model.Point; -import com.iqser.red.service.redaction.v1.model.Rectangle; - -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - @Slf4j @Data +@CompiledJson @NoArgsConstructor @JsonIgnoreProperties({"empty"}) public class TextPositionSequence implements CharSequence { @@ -27,7 +28,6 @@ public class TextPositionSequence implements CharSequence { private float x1; private float x2; - public TextPositionSequence(int page) { this.page = page; @@ -112,6 +112,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getX1() { if (textPositions.get(0).getRotation() == 90) { @@ -123,6 +124,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getX2() { if (textPositions.get(0).getRotation() == 90) { @@ -135,6 +137,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getRotationAdjustedY() { return textPositions.get(0).getY(); @@ -142,6 +145,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getRotationAdjustedX() { return textPositions.get(0).getXDirAdj(); @@ -149,6 +153,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getY1() { if (textPositions.get(0).getRotation() == 90) { @@ -160,6 +165,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getY2() { if (textPositions.get(0).getRotation() == 90) { @@ -171,6 +177,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getTextHeight() { return textPositions.get(0).getHeightDir() + 2; @@ -178,6 +185,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getHeight() { return getY2() - getY1(); @@ -185,6 +193,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getWidth() { return getX2() - getX1(); @@ -192,6 +201,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public String getFont() { return textPositions.get(0).getFontName().toLowerCase().replaceAll(",bold", "").replaceAll(",italic", ""); @@ -199,6 +209,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public String getFontStyle() { String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(); @@ -217,6 +228,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getFontSize() { return textPositions.get(0).getFontSizeInPt(); @@ -224,6 +236,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public float getSpaceWidth() { return textPositions.get(0).getWidthOfSpace(); @@ -231,6 +244,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public int getRotation() { return textPositions.get(0).getRotation(); @@ -238,6 +252,7 @@ public class TextPositionSequence implements CharSequence { @JsonIgnore + @JsonAttribute(ignore = true) public Rectangle getRectangle() { log.debug("Page: '{}', Word: '{}', Rotation: '{}', textRotation {}", page, toString(), textPositions.get(0).getRotation(), textPositions.get(0).getDir()); @@ -292,11 +307,11 @@ public class TextPositionSequence implements CharSequence { posXEnd = textPositions.get(textPositions.size() - 1).getYDirAdj() - height; } else if (textPositions.get(0).getRotation() == 180 && textPositions.get(0).getDir() == 180f) { - posXInit = textPositions.get(0).getPageWidth() - getX1() +1; + posXInit = textPositions.get(0).getPageWidth() - getX1() + 1; posXEnd = textPositions.get(0).getPageWidth() - getX2() + textPositions.get(0).getWidthDirAdj() - textPositions.get(textPositions.size() - 1) .getWidthDirAdj() - 3; posYInit = textPositions.get(0).getYDirAdj() - height + 2; - posYEnd = textPositions.get(textPositions.size() - 1) + posYEnd = textPositions.get(textPositions.size() - 1) .getYDirAdj() - height + 2; } else { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 35317fdf..69a342ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import com.dslplatform.json.CompiledJson; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -7,6 +8,7 @@ import lombok.NoArgsConstructor; @Data @Builder +@CompiledJson @NoArgsConstructor @AllArgsConstructor public class Image implements ReasonHolder { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java index d4ed9335..b20687b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/PdfImage.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.NonNull; @@ -13,6 +14,7 @@ import java.awt.image.BufferedImage; public class PdfImage { @JsonIgnore + @JsonAttribute(ignore = true) private BufferedImage image; @NonNull private RedRectangle2D position; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java index 6d6c5806..182658a5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/RedRectangle2D.java @@ -1,12 +1,15 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data +@CompiledJson @NoArgsConstructor @AllArgsConstructor public class RedRectangle2D { @@ -19,6 +22,7 @@ public class RedRectangle2D { private double height; @JsonIgnore + @JsonAttribute(ignore = true) public boolean isEmpty() { return width <= 0.0f || height <= 0.0f; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java index 3d89445a..60565609 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; @@ -9,12 +10,12 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.regex.Pattern; public class SearchableText { @JsonIgnore + @JsonAttribute(ignore = true) private transient String stringRepresentation; private final List sequences = new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java index f756568f..1c0b4335 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Classification.java @@ -3,9 +3,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; import java.util.HashMap; import java.util.Map; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class Classification { private Map probabilities = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java index 9fdabb52..82e30469 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/FilterGeometry.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class FilterGeometry { private ImageSize imageSize; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java index 49469e34..068333c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Filters.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class Filters { private FilterGeometry geometry; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java index 6ea063a2..86096664 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Geometry.java @@ -1,9 +1,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class Geometry { private float width; private float height; -} \ No newline at end of file +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java index c015e583..0d0a0817 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageFormat.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class ImageFormat { private float quotient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java index 57a29ae3..58b3997a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageMetadata.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class ImageMetadata { private Classification classification; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java index bf1ed455..4cf2b827 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageServiceResponse.java @@ -1,14 +1,16 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; +import com.dslplatform.json.JsonAttribute; +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + import java.util.ArrayList; import java.util.List; -import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonProperty; - -import lombok.Data; - @Data +@CompiledJson public class ImageServiceResponse { private String dossierId; @@ -16,6 +18,7 @@ public class ImageServiceResponse { @JsonProperty(value = "imageMetadata") @JsonAlias("data") + @JsonAttribute(alternativeNames = {"imageMetadata"}) private List data = new ArrayList<>(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java index 06c04440..a3a9e290 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/ImageSize.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class ImageSize { private float quotient; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java index 3422f6aa..64e0d130 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Position.java @@ -1,12 +1,14 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class Position { private float x1; private float x2; private float y1; private float y2; private int pageNumber; -} \ No newline at end of file +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Probability.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Probability.java index c7cf13c1..c2ca3f20 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Probability.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/image/Probability.java @@ -1,8 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.model.image; +import com.dslplatform.json.CompiledJson; import lombok.Data; @Data +@CompiledJson public class Probability { private boolean unconfident; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 02f31165..4fab1547 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -6,6 +6,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.annotations.entity import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; @@ -75,6 +76,7 @@ public class AnalyzeService { } List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); + sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); Text text = new Text(pageCount, sectionTexts); @@ -131,7 +133,7 @@ public class AnalyzeService { if (redactionServiceSettings.isNerServiceEnabled()) { nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } else { - nerEntities = NerEntities.builder().build(); + nerEntities = new NerEntities(); } List reanalysisSections = text.getSectionTexts() @@ -177,7 +179,7 @@ public class AnalyzeService { if (redactionServiceSettings.isNerServiceEnabled()) { nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } else { - nerEntities = NerEntities.builder().build(); + nerEntities = new NerEntities(); } dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); @@ -189,7 +191,7 @@ public class AnalyzeService { List redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId()); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, legalBasis, dictionary.getVersion() + var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, convert(legalBasis), dictionary.getVersion() .getDossierTemplateVersion(), dictionary.getVersion() .getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -286,6 +288,9 @@ public class AnalyzeService { .map(ManualForceRedaction::getAnnotationId)))).collect(Collectors.toSet()); } + public List convert(List legalBasis) { + return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); + } public Image convert(RedactionLogEntry entry) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 1ba4c7f0..f5a696f6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -69,7 +69,7 @@ public class RedactionLogMergeService { } private RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, ManualRedactions manualRedactions, - Set excludedPages, List types, Colors colors) { + Set excludedPages, List types, Colors colors) { var skippedImportedRedactions = new HashSet<>(); @@ -93,7 +93,7 @@ public class RedactionLogMergeService { skippedImportedRedactions.add(entry.getId()); } - entry.setComments(manualRedactions.getComments().get(entry.getId())); + entry.setComments(convert(manualRedactions.getComments().get(entry.getId()))); if (excludedPages != null && !excludedPages.isEmpty()) { entry.getPositions().forEach(pos -> { @@ -335,7 +335,7 @@ public class RedactionLogMergeService { if (shouldCreateManualEntry(manualRedactionEntry)) { RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), colors, types); redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions())); - redactionLogEntry.setComments(comments.get(manualRedactionEntry.getAnnotationId())); + redactionLogEntry.setComments(convert(comments.get(manualRedactionEntry.getAnnotationId()))); redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore()); redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter()); @@ -348,6 +348,11 @@ public class RedactionLogMergeService { return redactionLogEntries; } + private List convert(List comments) { + return comments.stream().map(c -> new RedactionLogComment(c.getId(), c.getUser(), c.getText(), c.getAnnotationId(), + c.getFileId(), c.getDate(), c.getSoftDeletedTime())).collect(Collectors.toList()); + } + private List convertPositions( List positions) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java index 7ef14ffd..706562a0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java @@ -46,7 +46,7 @@ public class SectionGridCreatorService { .getRectanglesPerPage() .computeIfAbsent(page, (x) -> new ArrayList<>()) .add(new SectionRectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock - .getHeight(), i + 1, paragraph.getPageBlocks().size())); + .getHeight(), i + 1, paragraph.getPageBlocks().size(),null)); } else if (textBlock instanceof Table) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java index 0e7448e4..fb4b0e81 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java @@ -79,7 +79,7 @@ public class SectionTextBuilderService { .get(0) .getSequences() .get(0) - .getPage()); + .getPage(), null); sectionText.getSectionAreas().add(sectionArea); sectionText.getTextBlocks().addAll(cell.getTextBlocks()); @@ -141,7 +141,7 @@ public class SectionTextBuilderService { .get(0) .getSequences() .get(0) - .getPage()); + .getPage(), null); sectionText.getTextBlocks().addAll(cell.getTextBlocks()); sectionText.getSectionAreas().add(sectionArea); @@ -171,7 +171,7 @@ public class SectionTextBuilderService { SectionText sectionText = new SectionText(); for (TextBlock paragraphTextBlock : paragraphTextBlocks) { SectionArea sectionArea = new SectionArea(new Point(paragraphTextBlock.getMinX(), paragraphTextBlock.getMinY()), paragraphTextBlock - .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage()); + .getWidth(), paragraphTextBlock.getHeight(), paragraphTextBlock.getPage(), null); sectionText.getSectionAreas().add(sectionArea); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java index ca7b0015..eb705a97 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/ImageService.java @@ -36,7 +36,7 @@ public class ImageService { ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class); Map> images = new HashMap<>(); - imageServiceResponse.getData().stream().forEach(imageMetadata -> { + imageServiceResponse.getData().forEach(imageMetadata -> { var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER; images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>()) .add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification, imageMetadata.getPosition().getPageNumber())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index a3c2ed78..ffd0e004 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -1,5 +1,27 @@ package com.iqser.red.service.redaction.v1.server.segmentation; +import com.iqser.red.service.redaction.v1.server.classification.model.Document; +import com.iqser.red.service.redaction.v1.server.classification.model.Page; +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; +import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; +import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService; +import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; +import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService; +import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.pdfbox.io.MemoryUsageSetting; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.springframework.stereotype.Service; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -13,38 +35,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.SystemUtils; -import org.apache.pdfbox.io.MemoryUsageSetting; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.pdmodel.PDPage; -import org.apache.pdfbox.pdmodel.common.PDRectangle; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.server.classification.model.Document; -import com.iqser.red.service.redaction.v1.server.classification.model.Page; -import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService; -import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService; -import com.iqser.red.service.redaction.v1.server.memory.MemoryStats; -import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; -import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; -import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; -import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService; -import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - @Slf4j @Service @RequiredArgsConstructor public class PdfSegmentationService { - private final static int MAX_PAGES_BEFORE_GC = 250; - private final RulingCleaningService rulingCleaningService; private final TableExtractionService tableExtractionService; private final BlockificationService blockificationService; @@ -60,11 +55,10 @@ public class PdfSegmentationService { //create tempFile File tempFile; - if(SystemUtils.IS_OS_UNIX) { + if (SystemUtils.IS_OS_UNIX) { FileAttribute> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")); tempFile = Files.createTempFile("document", ".pdf", attr).toFile(); - } - else { + } else { tempFile = Files.createTempFile("document", ".pdf").toFile(); tempFile.setReadable(true, true); tempFile.setWritable(true, true); @@ -78,15 +72,12 @@ public class PdfSegmentationService { Document document = new Document(); List pages = new ArrayList<>(); - pdDocument = reinitializePDDocument(tempFile, null); + pdDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupMixed(1024 * 1024 * 64)); + pdDocument.setAllSecurityToBeRemoved(true); long pageCount = pdDocument.getNumberOfPages(); for (int pageNumber = 1; pageNumber <= pageCount; pageNumber++) { - if (pageNumber % MAX_PAGES_BEFORE_GC == 0) { - pdDocument = reinitializePDDocument(tempFile, pdDocument); - } - PDFLinesTextStripper stripper = new PDFLinesTextStripper(); PDPage pdPage = pdDocument.getPage(pageNumber - 1); stripper.setPageNumber(pageNumber); @@ -107,6 +98,7 @@ public class PdfSegmentationService { Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings .getVertical()); + PDRectangle cropbox = pdPage.getCropBox(); float cropboxArea = cropbox.getHeight() * cropbox.getWidth(); page.setCropBoxArea(cropboxArea); @@ -134,11 +126,10 @@ public class PdfSegmentationService { sectionsBuilderService.buildSections(document); sectionsBuilderService.addImagesToSections(document); - pdDocument = reinitializePDDocument(tempFile, pdDocument); IOUtils.close(pdDocument); - if(!tempFile.delete()){ + if (!tempFile.delete()) { log.warn("Could not delete tmp file"); } @@ -152,23 +143,6 @@ public class PdfSegmentationService { } - private PDDocument reinitializePDDocument(File tempFile, PDDocument pdDocument) throws IOException { - - if (pdDocument != null) { - pdDocument.close(); - } - System.runFinalization(); - System.gc(); - - MemoryStats.printMemoryStats(); - - var newPDDocument = PDDocument.load(tempFile, MemoryUsageSetting.setupTempFileOnly()); - newPDDocument.setAllSecurityToBeRemoved(true); - - return newPDDocument; - } - - private void increaseDocumentStatistics(Page page, Document document) { if (!page.isLandscape()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 4ac357fb..62bf9b1e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -1,12 +1,7 @@ package com.iqser.red.service.redaction.v1.server.storage; -import java.io.IOException; -import java.io.InputStream; - -import org.springframework.core.io.InputStreamResource; -import org.springframework.stereotype.Service; - -import com.fasterxml.jackson.databind.ObjectMapper; +import com.dslplatform.json.DslJson; +import com.dslplatform.json.runtime.Settings; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.model.ImportedRedactions; import com.iqser.red.service.redaction.v1.model.RedactionLog; @@ -16,20 +11,26 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; - import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; @Slf4j @Service @RequiredArgsConstructor public class RedactionStorageService { - private final ObjectMapper objectMapper; private final StorageService storageService; + private final DslJson dslJson = new DslJson<>(Settings.basicSetup()); + @SneakyThrows public InputStream getStoredObject(String storageId) { @@ -40,12 +41,13 @@ public class RedactionStorageService { @SneakyThrows public void storeObject(String dossierId, String fileId, FileType fileType, Object any) { - - storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), objectMapper.writeValueAsBytes(any)); + var baos = new ByteArrayOutputStream(); + dslJson.serialize(any, baos); + storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), baos.toByteArray()); } @SneakyThrows - public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream){ + public void storeObject(String dossierId, String fileId, FileType fileType, InputStream inputStream) { storageService.storeObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType), inputStream); } @@ -62,7 +64,7 @@ public class RedactionStorageService { } try { - return objectMapper.readValue(inputStreamResource.getInputStream(), ImportedRedactions.class); + return dslJson.deserialize(ImportedRedactions.class, inputStreamResource.getInputStream()); } catch (IOException e) { throw new RuntimeException("Could not imported redactions", e); } @@ -80,7 +82,7 @@ public class RedactionStorageService { } try { - return objectMapper.readValue(inputStreamResource.getInputStream(), RedactionLog.class); + return dslJson.deserialize(RedactionLog.class, inputStreamResource.getInputStream()); } catch (IOException e) { throw new RuntimeException("Could not convert RedactionLog", e); } @@ -98,7 +100,7 @@ public class RedactionStorageService { } try { - return objectMapper.readValue(inputStreamResource.getInputStream(), Text.class); + return dslJson.deserialize(Text.class, inputStreamResource.getInputStream()); } catch (IOException e) { throw new RuntimeException("Could not convert Text", e); } @@ -115,7 +117,7 @@ public class RedactionStorageService { } try { - return objectMapper.readValue(inputStreamResource.getInputStream(), NerEntities.class); + return dslJson.deserialize(NerEntities.class, inputStreamResource.getInputStream()); } catch (IOException e) { throw new RuntimeException("Could not convert NER Entities", e); } @@ -126,7 +128,7 @@ public class RedactionStorageService { try { var sectionGrid = storageService.getObject(StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID)); - return objectMapper.readValue(sectionGrid.getInputStream(), SectionGrid.class); + return dslJson.deserialize(SectionGrid.class, sectionGrid.getInputStream()); } catch (StorageObjectDoesNotExist e) { throw new NotFoundException("Section Grid is not available."); } catch (IOException e) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java index cb7cfde5..cf608064 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java @@ -1,9 +1,9 @@ package com.iqser.red.service.redaction.v1.server.tableextraction.model; +import com.dslplatform.json.JsonAttribute; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Orientation; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -33,11 +33,13 @@ public abstract class AbstractTextContainer { } @JsonIgnore + @JsonAttribute(ignore = true) public float getHeight() { return maxY - minY; } - + @JsonIgnore + @JsonAttribute(ignore = true) public float getWidth() { return maxX - minX; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java index b68248aa..b82da1c5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java @@ -12,6 +12,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.util.*; +import java.util.stream.Collectors; public class FileSystemBackedStorageService implements StorageService{ @@ -51,6 +52,12 @@ public class FileSystemBackedStorageService implements StorageService{ return new ArrayList<>(dataMap.keySet()); } + + public List listFilePaths(){ + return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList()); + } + + @SneakyThrows @Override public void storeObject(String objectId, byte[] data) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 99dbe865..2403181f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -21,7 +21,6 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.controller.RedactionController; -import com.iqser.red.service.redaction.v1.server.memory.MemoryStats; import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService; import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; @@ -243,7 +242,6 @@ public class RedactionIntegrationTest { public void test270Rotated() { AnalyzeRequest request = prepareStorage("files/Minimal Examples/270Rotated.pdf"); - MemoryStats.printMemoryStats(); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); assertThat(result).isNotNull(); @@ -255,7 +253,6 @@ public class RedactionIntegrationTest { public void testLargeScannedFileOOM() { AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf"); - MemoryStats.printMemoryStats(); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); assertThat(result).isNotNull(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index dc1cebc6..987da9d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -124,7 +124,7 @@ public class AnnotationService { annotations.add(annotation); if (redactionLogEntry.getComments() != null) { - for (Comment comment : redactionLogEntry.getComments()) { + for (RedactionLogComment comment : redactionLogEntry.getComments()) { PDAnnotationText txtAnnot = new PDAnnotationText(); txtAnnot.setAnnotationName(String.valueOf(comment.getId())); txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation