Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Philipp Schramm
|
0701552e5c | ||
|
Philipp Schramm
|
be50483369 |
@@ -3,12 +3,6 @@ set -e
|
||||
|
||||
SERVICE_NAME=$1
|
||||
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
-f ${bamboo_build_working_directory}/$SERVICE_NAME-v1/pom.xml \
|
||||
--no-transfer-progress \
|
||||
clean install \
|
||||
-Djava.security.egd=file:/dev/./urandomelse
|
||||
|
||||
echo "dependency-check:aggregate"
|
||||
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
|
||||
--no-transfer-progress \
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<artifactId>platform-docker-dependency</artifactId>
|
||||
<version>1.1.0</version>
|
||||
<relativePath />
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
<parent>
|
||||
<artifactId>platform-dependency</artifactId>
|
||||
<groupId>com.iqser.red</groupId>
|
||||
<version>1.6.0</version>
|
||||
<relativePath />
|
||||
<version>1.1.7</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<artifactId>redaction-service-api-v1</artifactId>
|
||||
|
||||
<properties>
|
||||
<persistence-service.version>1.23.0</persistence-service.version>
|
||||
<persistence-service.version>0.135.0</persistence-service.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
-1
@@ -26,7 +26,6 @@ public class AnalyzeRequest {
|
||||
private String dossierTemplateId;
|
||||
private ManualRedactions manualRedactions;
|
||||
private OffsetDateTime lastProcessed;
|
||||
private int analysisNumber;
|
||||
|
||||
@Builder.Default
|
||||
private Set<Integer> excludedPages = new HashSet<>();
|
||||
|
||||
-1
@@ -28,7 +28,6 @@ public class AnalyzeResult {
|
||||
private boolean wasReanalyzed;
|
||||
|
||||
private int analysisVersion;
|
||||
private int analysisNumber;
|
||||
|
||||
private ManualRedactions manualRedactions;
|
||||
|
||||
|
||||
-1
@@ -13,7 +13,6 @@ import lombok.NoArgsConstructor;
|
||||
@NoArgsConstructor
|
||||
public class Change {
|
||||
|
||||
private int analysisNumber;
|
||||
private ChangeType type;
|
||||
private OffsetDateTime dateTime;
|
||||
}
|
||||
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
@Builder
|
||||
public class ManualChange {
|
||||
|
||||
private AnnotationStatus annotationStatus;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private OffsetDateTime processedDate;
|
||||
private String userId;
|
||||
private Map<String, Object> propertyChanges = new HashMap<>();
|
||||
|
||||
public boolean isProcessed() {
|
||||
return processedDate != null;
|
||||
}
|
||||
|
||||
public static ManualChange from(BaseAnnotation baseAnnotation) {
|
||||
ManualChange manualChange = new ManualChange();
|
||||
manualChange.annotationStatus = baseAnnotation.getStatus();
|
||||
manualChange.processedDate = baseAnnotation.getProcessedDate();
|
||||
manualChange.userId = baseAnnotation.getUser();
|
||||
return manualChange;
|
||||
}
|
||||
|
||||
public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) {
|
||||
this.manualRedactionType = manualRedactionType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ManualChange withChange(String property, Object value) {
|
||||
this.propertyChanges.put(property, value);
|
||||
return this;
|
||||
}
|
||||
|
||||
}
|
||||
+1
-9
@@ -1,13 +1,5 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
public enum ManualRedactionType {
|
||||
ADD_LOCALLY,
|
||||
ADD_TO_DICTIONARY,
|
||||
REMOVE_LOCALLY,
|
||||
REMOVE_FROM_DICTIONARY,
|
||||
FORCE_REDACT,
|
||||
FORCE_HINT,
|
||||
RECATEGORIZE,
|
||||
LEGAL_BASIS_CHANGE,
|
||||
RESIZE
|
||||
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE
|
||||
}
|
||||
|
||||
-6
@@ -18,11 +18,6 @@ public class RedactionLog {
|
||||
*/
|
||||
private long analysisVersion;
|
||||
|
||||
/**
|
||||
* Which analysis created this redactionLog.
|
||||
*/
|
||||
private int analysisNumber;
|
||||
|
||||
private List<RedactionLogEntry> redactionLogEntry;
|
||||
private List<LegalBasis> legalBasis;
|
||||
|
||||
@@ -31,5 +26,4 @@ public class RedactionLog {
|
||||
private long rulesVersion = -1;
|
||||
private long legalBasisVersion = -1;
|
||||
|
||||
|
||||
}
|
||||
|
||||
+24
-26
@@ -2,9 +2,16 @@ package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import lombok.*;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
@Data
|
||||
@@ -21,18 +28,27 @@ public class RedactionLogEntry {
|
||||
private int matchedRule;
|
||||
private boolean rectangle;
|
||||
private String legalBasis;
|
||||
|
||||
private boolean redacted;
|
||||
private boolean isHint;
|
||||
private boolean isRecommendation;
|
||||
|
||||
private boolean hasBeenResized;
|
||||
private boolean hasBeenRecategorized;
|
||||
private boolean hasLegalBasisChanged;
|
||||
private boolean hasBeenForced;
|
||||
private boolean hasBeenRemovedByManualOverride;
|
||||
|
||||
private String section;
|
||||
private float[] color;
|
||||
|
||||
@Builder.Default
|
||||
private List<Rectangle> positions = new ArrayList<>();
|
||||
private int sectionNumber;
|
||||
|
||||
private boolean manual;
|
||||
private AnnotationStatus status;
|
||||
private ManualRedactionType manualRedactionType;
|
||||
private String manualRedactionUserId;
|
||||
private boolean isDictionaryEntry;
|
||||
|
||||
private String textBefore;
|
||||
private String textAfter;
|
||||
@@ -46,39 +62,21 @@ public class RedactionLogEntry {
|
||||
private boolean isImage;
|
||||
private boolean imageHasTransparency;
|
||||
|
||||
private boolean isDictionaryEntry;
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
private boolean excluded;
|
||||
|
||||
private String recategorizationType;
|
||||
private String legalBasisChangeValue;
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
@Builder.Default
|
||||
private List<Change> changes = new ArrayList<>();
|
||||
|
||||
@EqualsAndHashCode.Exclude
|
||||
@Builder.Default
|
||||
private List<ManualChange> manualChanges = new ArrayList<>();
|
||||
|
||||
private Set<Engine> engines = new HashSet<>();
|
||||
private Set<Engine> engines= new HashSet<>();
|
||||
|
||||
private Set<String> reference = new HashSet<>();
|
||||
|
||||
public boolean lastChangeIsRemoved() {
|
||||
return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false);
|
||||
}
|
||||
|
||||
public boolean isLocalManualRedaction() {
|
||||
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY &&
|
||||
mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
|
||||
}
|
||||
|
||||
public boolean isManuallyRemoved() {
|
||||
return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY &&
|
||||
mc.getAnnotationStatus() == AnnotationStatus.APPROVED);
|
||||
}
|
||||
|
||||
private <T> Optional<T> last(List<T> list) {
|
||||
return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-2
@@ -7,7 +7,6 @@ import lombok.Data;
|
||||
import lombok.NonNull;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@@ -17,7 +16,7 @@ public class Page {
|
||||
@NonNull
|
||||
private List<AbstractTextContainer> textBlocks;
|
||||
|
||||
private List<PdfImage> images = new ArrayList<>();
|
||||
private List<PdfImage> images;
|
||||
|
||||
private Rectangle bodyTextFrame;
|
||||
|
||||
|
||||
+4
-1
@@ -1,9 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecogintionEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
|
||||
@@ -12,5 +16,4 @@ public interface EntityRecognitionClient {
|
||||
|
||||
@PostMapping(value = "/find_authors", produces = MediaType.APPLICATION_JSON_VALUE, consumes = MediaType.APPLICATION_JSON_VALUE)
|
||||
NerEntities findAuthors(EntityRecognitionRequest entityRecognitionRequest);
|
||||
|
||||
}
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import org.springframework.cloud.openfeign.FeignClient;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
@FeignClient(name = "ImageClassificationResource", url = "${image-service.url}")
|
||||
public interface ImageClassificationClient {
|
||||
|
||||
@PostMapping(value = "/process_full_img", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
ImageClassificationResponse classify(@RequestBody MultipartFile file);
|
||||
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.client;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ImageClassificationResponse {
|
||||
|
||||
private String category;
|
||||
}
|
||||
+3
-3
@@ -88,7 +88,7 @@ public class RedactionController implements RedactionResource {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
@@ -116,7 +116,7 @@ public class RedactionController implements RedactionResource {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
try {
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
Document classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
|
||||
storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
@@ -145,7 +145,7 @@ public class RedactionController implements RedactionResource {
|
||||
try {
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(redactionRequest
|
||||
.getDossierId(), redactionRequest.getFileId(), FileType.ORIGIN));
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, null);
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, true);
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
}
|
||||
|
||||
-377
@@ -1,377 +0,0 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package com.iqser.red.service.redaction.v1.server.parsing;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.WeakHashMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.fontbox.ttf.TrueTypeFont;
|
||||
import org.apache.fontbox.util.BoundingBox;
|
||||
|
||||
import org.apache.pdfbox.contentstream.PDFStreamEngine;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.font.encoding.GlyphList;
|
||||
import org.apache.pdfbox.pdmodel.common.PDRectangle;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDCIDFontType2;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDSimpleFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDTrueTypeFont;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType0Font;
|
||||
import org.apache.pdfbox.pdmodel.font.PDType3Font;
|
||||
import org.apache.pdfbox.pdmodel.graphics.state.PDGraphicsState;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
import org.apache.pdfbox.util.Vector;
|
||||
import org.apache.pdfbox.contentstream.operator.DrawObject;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Concatenate;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Restore;
|
||||
import org.apache.pdfbox.contentstream.operator.state.Save;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetGraphicsStateParameters;
|
||||
import org.apache.pdfbox.contentstream.operator.state.SetMatrix;
|
||||
import org.apache.pdfbox.contentstream.operator.text.BeginText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.EndText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextHorizontalScaling;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextAdjusted;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowTextLineAndSpace;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveText;
|
||||
import org.apache.pdfbox.contentstream.operator.text.MoveTextSetLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.NextLine;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetCharSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextLeading;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRenderingMode;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetTextRise;
|
||||
import org.apache.pdfbox.contentstream.operator.text.SetWordSpacing;
|
||||
import org.apache.pdfbox.contentstream.operator.text.ShowText;
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.pdmodel.font.PDFontDescriptor;
|
||||
|
||||
/**
|
||||
* LEGACY text calculations which are known to be incorrect but are depended on by PDFTextStripper.
|
||||
*
|
||||
* This class exists only so that we don't break the code of users who have their own subclasses of
|
||||
* PDFTextStripper. It replaces the mostly empty implementation of showGlyph() in PDFStreamEngine
|
||||
* with a heuristic implementation which is backwards compatible.
|
||||
*
|
||||
* DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
* THIS CODE IS DELIBERATELY INCORRECT, USE PDFStreamEngine INSTEAD.
|
||||
*/
|
||||
@SuppressWarnings({"PMD", "checkstyle:all"})
|
||||
class LegacyPDFStreamEngine extends PDFStreamEngine
|
||||
{
|
||||
private static final Log LOG = LogFactory.getLog(LegacyPDFStreamEngine.class);
|
||||
|
||||
private int pageRotation;
|
||||
private PDRectangle pageSize;
|
||||
private Matrix translateMatrix;
|
||||
private final GlyphList glyphList;
|
||||
private final Map<COSDictionary, Float> fontHeightMap = new WeakHashMap<COSDictionary, Float>();
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
LegacyPDFStreamEngine() throws IOException
|
||||
{
|
||||
addOperator(new BeginText());
|
||||
addOperator(new Concatenate());
|
||||
addOperator(new DrawObject()); // special text version
|
||||
addOperator(new EndText());
|
||||
addOperator(new SetGraphicsStateParameters());
|
||||
addOperator(new Save());
|
||||
addOperator(new Restore());
|
||||
addOperator(new NextLine());
|
||||
addOperator(new SetCharSpacing());
|
||||
addOperator(new MoveText());
|
||||
addOperator(new MoveTextSetLeading());
|
||||
addOperator(new SetFontAndSize());
|
||||
addOperator(new ShowText());
|
||||
addOperator(new ShowTextAdjusted());
|
||||
addOperator(new SetTextLeading());
|
||||
addOperator(new SetMatrix());
|
||||
addOperator(new SetTextRenderingMode());
|
||||
addOperator(new SetTextRise());
|
||||
addOperator(new SetWordSpacing());
|
||||
addOperator(new SetTextHorizontalScaling());
|
||||
addOperator(new ShowTextLine());
|
||||
addOperator(new ShowTextLineAndSpace());
|
||||
|
||||
// load additional glyph list for Unicode mapping
|
||||
String path = "/org/apache/pdfbox/resources/glyphlist/additional.txt";
|
||||
InputStream input = GlyphList.class.getResourceAsStream(path);
|
||||
glyphList = new GlyphList(GlyphList.getAdobeGlyphList(), input);
|
||||
}
|
||||
|
||||
/**
|
||||
* This will initialize and process the contents of the stream.
|
||||
*
|
||||
* @param page the page to process
|
||||
* @throws java.io.IOException if there is an error accessing the stream.
|
||||
*/
|
||||
@Override
|
||||
public void processPage(PDPage page) throws IOException
|
||||
{
|
||||
this.pageRotation = page.getRotation();
|
||||
this.pageSize = page.getCropBox();
|
||||
|
||||
if (pageSize.getLowerLeftX() == 0 && pageSize.getLowerLeftY() == 0)
|
||||
{
|
||||
translateMatrix = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
// translation matrix for cropbox
|
||||
translateMatrix = Matrix.getTranslateInstance(-pageSize.getLowerLeftX(), -pageSize.getLowerLeftY());
|
||||
}
|
||||
super.processPage(page);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when a glyph is to be processed. The heuristic calculations here were originally
|
||||
* written by Ben Litchfield for PDFStreamEngine.
|
||||
*/
|
||||
@Override
|
||||
protected void showGlyph(Matrix textRenderingMatrix, PDFont font, int code,
|
||||
String unicode,
|
||||
Vector displacement)
|
||||
throws IOException
|
||||
{
|
||||
//
|
||||
// legacy calculations which were previously in PDFStreamEngine
|
||||
//
|
||||
// DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
// THIS CODE IS DELIBERATELY INCORRECT
|
||||
//
|
||||
|
||||
PDGraphicsState state = getGraphicsState();
|
||||
Matrix ctm = state.getCurrentTransformationMatrix();
|
||||
float fontSize = state.getTextState().getFontSize();
|
||||
float horizontalScaling = state.getTextState().getHorizontalScaling() / 100f;
|
||||
Matrix textMatrix = getTextMatrix();
|
||||
|
||||
float displacementX = displacement.getX();
|
||||
// the sorting algorithm is based on the width of the character. As the displacement
|
||||
// for vertical characters doesn't provide any suitable value for it, we have to
|
||||
// calculate our own
|
||||
if (font.isVertical())
|
||||
{
|
||||
displacementX = font.getWidth(code) / 1000;
|
||||
// there may be an additional scaling factor for true type fonts
|
||||
TrueTypeFont ttf = null;
|
||||
if (font instanceof PDTrueTypeFont)
|
||||
{
|
||||
ttf = ((PDTrueTypeFont)font).getTrueTypeFont();
|
||||
}
|
||||
else if (font instanceof PDType0Font)
|
||||
{
|
||||
PDCIDFont cidFont = ((PDType0Font)font).getDescendantFont();
|
||||
if (cidFont instanceof PDCIDFontType2)
|
||||
{
|
||||
ttf = ((PDCIDFontType2)cidFont).getTrueTypeFont();
|
||||
}
|
||||
}
|
||||
if (ttf != null && ttf.getUnitsPerEm() != 1000)
|
||||
{
|
||||
displacementX *= 1000f / ttf.getUnitsPerEm();
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// legacy calculations which were previously in PDFStreamEngine
|
||||
//
|
||||
// DO NOT USE THIS CODE UNLESS YOU ARE WORKING WITH PDFTextStripper.
|
||||
// THIS CODE IS DELIBERATELY INCORRECT
|
||||
//
|
||||
|
||||
// (modified) combined displacement, this is calculated *without* taking the character
|
||||
// spacing and word spacing into account, due to legacy code in TextStripper
|
||||
float tx = displacementX * fontSize * horizontalScaling;
|
||||
float ty = displacement.getY() * fontSize;
|
||||
|
||||
// (modified) combined displacement matrix
|
||||
Matrix td = Matrix.getTranslateInstance(tx, ty);
|
||||
|
||||
// (modified) text rendering matrix
|
||||
Matrix nextTextRenderingMatrix = td.multiply(textMatrix).multiply(ctm); // text space -> device space
|
||||
float nextX = nextTextRenderingMatrix.getTranslateX();
|
||||
float nextY = nextTextRenderingMatrix.getTranslateY();
|
||||
|
||||
// (modified) width and height calculations
|
||||
float dxDisplay = nextX - textRenderingMatrix.getTranslateX();
|
||||
Float fontHeight = fontHeightMap.get(font.getCOSObject());
|
||||
if (fontHeight == null)
|
||||
{
|
||||
fontHeight = computeFontHeight(font);
|
||||
fontHeightMap.put(font.getCOSObject(), fontHeight);
|
||||
}
|
||||
float dyDisplay = fontHeight * textRenderingMatrix.getScalingFactorY();
|
||||
|
||||
//
|
||||
// start of the original method
|
||||
//
|
||||
|
||||
// Note on variable names. There are three different units being used in this code.
|
||||
// Character sizes are given in glyph units, text locations are initially given in text
|
||||
// units, and we want to save the data in display units. The variable names should end with
|
||||
// Text or Disp to represent if the values are in text or disp units (no glyph units are
|
||||
// saved).
|
||||
|
||||
float glyphSpaceToTextSpaceFactor = 1 / 1000f;
|
||||
if (font instanceof PDType3Font)
|
||||
{
|
||||
glyphSpaceToTextSpaceFactor = font.getFontMatrix().getScaleX();
|
||||
}
|
||||
|
||||
float spaceWidthText = 0;
|
||||
try
|
||||
{
|
||||
// to avoid crash as described in PDFBOX-614, see what the space displacement should be
|
||||
spaceWidthText = font.getSpaceWidth() * glyphSpaceToTextSpaceFactor;
|
||||
}
|
||||
catch (Throwable exception)
|
||||
{
|
||||
LOG.warn(exception, exception);
|
||||
}
|
||||
|
||||
if (spaceWidthText == 0)
|
||||
{
|
||||
spaceWidthText = font.getAverageFontWidth() * glyphSpaceToTextSpaceFactor;
|
||||
// the average space width appears to be higher than necessary so make it smaller
|
||||
spaceWidthText *= .80f;
|
||||
}
|
||||
if (spaceWidthText == 0)
|
||||
{
|
||||
spaceWidthText = 1.0f; // if could not find font, use a generic value
|
||||
}
|
||||
|
||||
// the space width has to be transformed into display units
|
||||
float spaceWidthDisplay = spaceWidthText * textRenderingMatrix.getScalingFactorX();
|
||||
|
||||
// use our additional glyph list for Unicode mapping
|
||||
String unicodeMapping = font.toUnicode(code, glyphList);
|
||||
|
||||
// when there is no Unicode mapping available, Acrobat simply coerces the character code
|
||||
// into Unicode, so we do the same. Subclasses of PDFStreamEngine don't necessarily want
|
||||
// this, which is why we leave it until this point in PDFTextStreamEngine.
|
||||
if (unicodeMapping == null)
|
||||
{
|
||||
if (font instanceof PDSimpleFont)
|
||||
{
|
||||
char c = (char) code;
|
||||
unicodeMapping = new String(new char[] { c });
|
||||
}
|
||||
else
|
||||
{
|
||||
// Acrobat doesn't seem to coerce composite font's character codes, instead it
|
||||
// skips them. See the "allah2.pdf" TestTextStripper file.
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// adjust for cropbox if needed
|
||||
Matrix translatedTextRenderingMatrix;
|
||||
if (translateMatrix == null)
|
||||
{
|
||||
translatedTextRenderingMatrix = textRenderingMatrix;
|
||||
}
|
||||
else
|
||||
{
|
||||
translatedTextRenderingMatrix = Matrix.concatenate(translateMatrix, textRenderingMatrix);
|
||||
nextX -= pageSize.getLowerLeftX();
|
||||
nextY -= pageSize.getLowerLeftY();
|
||||
}
|
||||
|
||||
processTextPosition(new TextPosition(pageRotation, pageSize.getWidth(),
|
||||
pageSize.getHeight(), translatedTextRenderingMatrix, nextX, nextY,
|
||||
Math.abs(dyDisplay), dxDisplay,
|
||||
Math.abs(spaceWidthDisplay), unicodeMapping, new int[] { code }, font,
|
||||
fontSize,
|
||||
(int)(fontSize * textMatrix.getScalingFactorX())));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the font height. Override this if you want to use own calculations.
|
||||
*
|
||||
* @param font the font.
|
||||
* @return the font height.
|
||||
*
|
||||
* @throws IOException if there is an error while getting the font bounding box.
|
||||
*/
|
||||
protected float computeFontHeight(PDFont font) throws IOException
|
||||
{
|
||||
BoundingBox bbox = font.getBoundingBox();
|
||||
if (bbox.getLowerLeftY() < Short.MIN_VALUE)
|
||||
{
|
||||
// PDFBOX-2158 and PDFBOX-3130
|
||||
// files by Salmat eSolutions / ClibPDF Library
|
||||
bbox.setLowerLeftY(- (bbox.getLowerLeftY() + 65536));
|
||||
}
|
||||
// 1/2 the bbox is used as the height todo: why?
|
||||
float glyphHeight = bbox.getHeight() / 2;
|
||||
|
||||
// sometimes the bbox has very high values, but CapHeight is OK
|
||||
PDFontDescriptor fontDescriptor = font.getFontDescriptor();
|
||||
if (fontDescriptor != null)
|
||||
{
|
||||
float capHeight = fontDescriptor.getCapHeight();
|
||||
if (Float.compare(capHeight, 0) != 0 &&
|
||||
(capHeight < glyphHeight || Float.compare(glyphHeight, 0) == 0))
|
||||
{
|
||||
glyphHeight = capHeight;
|
||||
}
|
||||
// PDFBOX-3464, PDFBOX-4480, PDFBOX-4553:
|
||||
// sometimes even CapHeight has very high value, but Ascent and Descent are ok
|
||||
float ascent = fontDescriptor.getAscent();
|
||||
float descent = fontDescriptor.getDescent();
|
||||
if (capHeight > ascent && ascent > 0 && descent < 0 &&
|
||||
((ascent - descent) / 2 < glyphHeight || Float.compare(glyphHeight, 0) == 0))
|
||||
{
|
||||
glyphHeight = (ascent - descent) / 2;
|
||||
}
|
||||
}
|
||||
|
||||
// transformPoint from glyph space -> text space
|
||||
float height;
|
||||
if (font instanceof PDType3Font)
|
||||
{
|
||||
height = font.getFontMatrix().transformPoint(0, glyphHeight).y;
|
||||
}
|
||||
else
|
||||
{
|
||||
height = glyphHeight / 1000;
|
||||
}
|
||||
|
||||
return height;
|
||||
}
|
||||
|
||||
/**
|
||||
* A method provided as an event interface to allow a subclass to perform some specific
|
||||
* functionality when text needs to be processed.
|
||||
*
|
||||
* @param text The text to be processed.
|
||||
*/
|
||||
protected void processTextPosition(TextPosition text)
|
||||
{
|
||||
// subclasses can override to provide specific functionality
|
||||
}
|
||||
}
|
||||
+32
-26
@@ -36,6 +36,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDPage;
|
||||
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
|
||||
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
|
||||
import org.apache.pdfbox.text.PDFTextStripper;
|
||||
import org.apache.pdfbox.text.TextPosition;
|
||||
import org.apache.pdfbox.util.Matrix;
|
||||
|
||||
@@ -124,9 +125,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
// The direction of vertical lines must always be from bottom to top for the table extraction algorithm.
|
||||
if (pos.getY() > path_y) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, path_y), new Point2D.Float((float) pos.getX(), (float) pos
|
||||
.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos.getX(), path_y)));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float(path_x, (float) pos.getY()), new Point2D.Float((float) pos
|
||||
.getX(), path_y)));
|
||||
}
|
||||
|
||||
path_x = (float) pos.getX();
|
||||
@@ -147,19 +150,25 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
Point2D p2 = transformPosition(x + width, y + height);
|
||||
|
||||
// Horizontal lines
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
|
||||
// Vertical lines, direction must always be from bottom to top for the table extraction algorithm.
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p1.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p2.getX(), (float) p2.getY()), new Point2D.Float((float) p2
|
||||
.getX(), (float) p1.getY())));
|
||||
}
|
||||
if (p2.getY() > p1.getY()) {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1.getX(), (float) p2.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p1.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p2.getY())));
|
||||
} else {
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1.getX(), (float) p1.getY())));
|
||||
graphicsPath.add(new Ruling(new Point2D.Float((float) p1.getX(), (float) p2.getY()), new Point2D.Float((float) p1
|
||||
.getX(), (float) p1.getY())));
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -183,9 +192,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
graphicsPath.clear();
|
||||
break;
|
||||
|
||||
// case OperatorName.DRAW_OBJECT:
|
||||
// processImageOperation(arguments);
|
||||
// break;
|
||||
case OperatorName.DRAW_OBJECT:
|
||||
processImageOperation(arguments);
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
@@ -202,7 +211,8 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
PDImageXObject image = (PDImageXObject) xobject;
|
||||
Matrix ctmNew = getGraphicsState().getCurrentTransformationMatrix();
|
||||
|
||||
Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew.getScaleY());
|
||||
Rectangle2D rect = new Rectangle2D.Float(ctmNew.getTranslateX(), ctmNew.getTranslateY(), ctmNew.getScaleX(), ctmNew
|
||||
.getScaleY());
|
||||
|
||||
// Memory Hack - sofReference kills me
|
||||
FieldUtils.writeField(image, "cachedImageSubsampling", -1, true);
|
||||
@@ -285,7 +295,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
|
||||
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
|
||||
.equals("\u00A0") || textPositions.get(i)
|
||||
.getUnicode()
|
||||
.equals("\t"))) {
|
||||
startIndex++;
|
||||
continue;
|
||||
}
|
||||
@@ -295,9 +307,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
@@ -308,9 +318,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
|
||||
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
|
||||
}
|
||||
startIndex = i;
|
||||
@@ -324,9 +332,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, i);
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
.equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
|
||||
|
||||
// Remove false sequence ends (whitespaces)
|
||||
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
|
||||
@@ -345,15 +351,15 @@ public class PDFLinesTextStripper extends PDFTextStripper {
|
||||
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
|
||||
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals(" ") || sublist.get(sublist.size() - 1)
|
||||
.getUnicode()
|
||||
.equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
|
||||
.equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0") || sublist.get(sublist.size() - 1).getUnicode().equals("\t"))) {
|
||||
sublist = sublist.subList(0, sublist.size() - 1);
|
||||
}
|
||||
|
||||
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\u00A0") || sublist.get(0).getUnicode().equals("\t")))) {
|
||||
.equals("\u00A0") || sublist.get(0)
|
||||
.getUnicode()
|
||||
.equals("\t")))) {
|
||||
if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
|
||||
.getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
|
||||
for (TextPosition t : sublist) {
|
||||
|
||||
-2181
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -254,7 +254,7 @@ public class TextPositionSequence implements CharSequence {
|
||||
posYInit = getY1();
|
||||
posYEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() - height + 4;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 270.0f) {
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.size() > 1) {
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
posYEnd = textPositions.get(0).getPageHeight() - getX2() - textPositions.get(0)
|
||||
.getWidth() - textPositions.get(textPositions.size() - 1).getWidth() - 1;
|
||||
@@ -262,11 +262,11 @@ public class TextPositionSequence implements CharSequence {
|
||||
posXEnd = textPositions.get(0).getPageWidth() - textPositions.get(textPositions.size() - 1)
|
||||
.getYDirAdj() + height;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.get(0).getDir() == 0.0f) {
|
||||
} else if (textPositions.get(0).getRotation() == 270 && textPositions.size() == 1) {
|
||||
posYInit = textPositions.get(0).getPageHeight() - textPositions.get(0).getYDirAdj() - 2;
|
||||
posYEnd = posYInit + 1;
|
||||
posXInit = textPositions.get(0).getXDirAdj();
|
||||
posXEnd = textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + 0.1f;
|
||||
posXEnd = posXInit + textPositions.get(0).getWidthDirAdj() + 0.1f;
|
||||
|
||||
} else if (textPositions.get(0).getRotation() == 0 && textPositions.get(0).getDir() == 270f) {
|
||||
posYInit = textPositions.get(0).getPageHeight() - getX1();
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Entities {
|
||||
|
||||
@Builder.Default
|
||||
private Set<Entity> entities = new HashSet<>();
|
||||
|
||||
@Builder.Default
|
||||
private Set<Entity> nerEntities = new HashSet<>();
|
||||
|
||||
}
|
||||
+2
-4
@@ -15,8 +15,8 @@ import java.util.Set;
|
||||
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
|
||||
public class Entity implements ReasonHolder {
|
||||
|
||||
private String word;
|
||||
private String type;
|
||||
private final String word;
|
||||
private final String type;
|
||||
private boolean redaction;
|
||||
private String redactionReason;
|
||||
private String legalBasis;
|
||||
@@ -42,8 +42,6 @@ public class Entity implements ReasonHolder {
|
||||
|
||||
private boolean isDossierDictionaryEntry;
|
||||
|
||||
private boolean ignored;
|
||||
|
||||
private Set<Engine> engines = new HashSet<>();
|
||||
|
||||
private Set<Entity> references = new HashSet<>();
|
||||
|
||||
-1
@@ -20,7 +20,6 @@ public class Image implements ReasonHolder {
|
||||
private int sectionNumber;
|
||||
private String section;
|
||||
private int page;
|
||||
private boolean ignored;
|
||||
private boolean hasTransparency;
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -16,7 +16,6 @@ public class PdfImage {
|
||||
private BufferedImage image;
|
||||
@NonNull
|
||||
private RedRectangle2D position;
|
||||
@NonNull
|
||||
private ImageType imageType;
|
||||
private boolean isAppendedToParagraph;
|
||||
private boolean hasTransparency;
|
||||
|
||||
+22
-128
@@ -8,11 +8,8 @@ import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
@@ -47,8 +44,6 @@ public class Section {
|
||||
|
||||
private Set<Entity> entities;
|
||||
|
||||
private Set<Entity> nerEntities;
|
||||
|
||||
// This still contains linebreaks etc.
|
||||
private String text;
|
||||
|
||||
@@ -72,102 +67,6 @@ public class Section {
|
||||
private List<FileAttribute> fileAttributes = new ArrayList<>();
|
||||
|
||||
|
||||
public void addAiEntities(String type, String asType) {
|
||||
|
||||
Set<Entity> entitiesOfType = nerEntities.stream()
|
||||
.filter(nerEntity -> nerEntity.getType().equals(type))
|
||||
.collect(Collectors.toSet());
|
||||
Set<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toSet());
|
||||
Set<Entity> found = EntitySearchUtils.find(searchText, values, asType, headline, sectionNumber, false, false, Engine.NER, true);
|
||||
EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
|
||||
Set<Entity> finalResult = new HashSet<>();
|
||||
|
||||
// Only keep Entities with correct offsets from AI Service.
|
||||
Iterator<Entity> itty = found.iterator();
|
||||
while (itty.hasNext()) {
|
||||
Entity current = itty.next();
|
||||
boolean foundSameOffsets = false;
|
||||
for (Entity entity : entitiesOfType) {
|
||||
if (entity.getStart().equals(current.getStart()) && entity.getEnd().equals(current.getEnd())) {
|
||||
foundSameOffsets = true;
|
||||
}
|
||||
}
|
||||
if (foundSameOffsets) {
|
||||
finalResult.add(current);
|
||||
}
|
||||
}
|
||||
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, finalResult, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
nerEntities.removeAll(entitiesOfType);
|
||||
}
|
||||
|
||||
|
||||
public void combineAiTypes(String startType, String combineTypes, int maxDistanceBetween, String asType,
|
||||
int minPartMatches, boolean allowDuplicateTypes) {
|
||||
|
||||
Set<String> combineSet = Set.of(combineTypes.split(","));
|
||||
|
||||
List<Entity> sorted = nerEntities.stream()
|
||||
.sorted(Comparator.comparing(Entity::getStart))
|
||||
.collect(Collectors.toList());
|
||||
Set<Entity> found = new HashSet<>();
|
||||
int start = -1;
|
||||
int lastEnd = -1;
|
||||
int numberOfMatchParts = 0;
|
||||
Set<String> foundParts = new HashSet<>();
|
||||
for (Entity entity : sorted) {
|
||||
if (entity.getType().equals(startType) && start == -1) {
|
||||
lastEnd = entity.getEnd();
|
||||
start = entity.getStart();
|
||||
foundParts.add(entity.getType());
|
||||
numberOfMatchParts++;
|
||||
} else if (!allowDuplicateTypes && foundParts.contains(entity.getType())) {
|
||||
if (numberOfMatchParts >= minPartMatches) {
|
||||
String value = searchText.substring(start, lastEnd);
|
||||
found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
|
||||
}
|
||||
start = -1;
|
||||
lastEnd = -1;
|
||||
numberOfMatchParts = 0;
|
||||
foundParts = new HashSet<>();
|
||||
if (entity.getType().equals(startType)) {
|
||||
lastEnd = entity.getEnd();
|
||||
start = entity.getStart();
|
||||
foundParts.add(entity.getType());
|
||||
numberOfMatchParts++;
|
||||
}
|
||||
} else if (entity.getType().equals(startType) && start != -1) {
|
||||
if (numberOfMatchParts >= minPartMatches) {
|
||||
String value = searchText.substring(start, lastEnd);
|
||||
found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
|
||||
}
|
||||
start = entity.getStart();
|
||||
lastEnd = entity.getEnd();
|
||||
numberOfMatchParts = 0;
|
||||
foundParts = new HashSet<>();
|
||||
foundParts.add(entity.getType());
|
||||
numberOfMatchParts++;
|
||||
} else if (start != -1 && combineSet.contains(entity.getType()) && entity.getStart() - lastEnd < maxDistanceBetween) {
|
||||
lastEnd = entity.getEnd();
|
||||
numberOfMatchParts++;
|
||||
foundParts.add(entity.getType());
|
||||
}
|
||||
}
|
||||
|
||||
if (numberOfMatchParts >= minPartMatches) {
|
||||
String value = searchText.substring(start, lastEnd);
|
||||
found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER));
|
||||
}
|
||||
|
||||
if (!found.isEmpty()) {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(entities);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean fileAttributeByIdEquals(@Argument(ArgumentType.FILE_ATTRIBUTE) String id,
|
||||
@Argument(ArgumentType.STRING) String value) {
|
||||
@@ -254,24 +153,17 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean aiMatchesType(@Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
return nerEntities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
|
||||
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@WhenCondition
|
||||
public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
|
||||
|
||||
return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
|
||||
return images.stream().anyMatch(image -> image.getType().equals(type));
|
||||
}
|
||||
|
||||
|
||||
@@ -283,8 +175,7 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group) {
|
||||
|
||||
@@ -293,8 +184,7 @@ public class Section {
|
||||
|
||||
|
||||
@ThenAction
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
|
||||
@Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
public void expandByRegEx(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.REGEX) String suffixPattern,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
|
||||
@Argument(ArgumentType.INTEGER) int group,
|
||||
@Argument(ArgumentType.REGEX) String valuePattern) {
|
||||
@@ -327,7 +217,7 @@ public class Section {
|
||||
String match = matcher.group(group);
|
||||
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis(), Engine.RULE);
|
||||
Set<Entity> expandedEntities = findEntities(entity.getWord() + match, type, false, entity.isRedaction(), entity.getMatchedRule(), entity.getRedactionReason(), entity.getLegalBasis());
|
||||
expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
|
||||
}
|
||||
}
|
||||
@@ -406,8 +296,7 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
public void ignore(String type) {
|
||||
|
||||
public void ignore(String type){
|
||||
entities.removeIf(entity -> entity.getType().equals(type));
|
||||
}
|
||||
|
||||
@@ -457,7 +346,7 @@ public class Section {
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null, Engine.RULE));
|
||||
expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -480,7 +369,7 @@ public class Section {
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null, Engine.RULE);
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
@@ -509,7 +398,7 @@ public class Section {
|
||||
public void addHintAnnotation(@Argument(ArgumentType.STRING) String value,
|
||||
@Argument(ArgumentType.TYPE) String asType) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
@@ -520,7 +409,7 @@ public class Section {
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
|
||||
}
|
||||
|
||||
@@ -537,7 +426,7 @@ public class Section {
|
||||
if (values != null) {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
@@ -590,7 +479,7 @@ public class Section {
|
||||
while (matcher.find()) {
|
||||
String match = matcher.group(group);
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
}
|
||||
}
|
||||
@@ -653,7 +542,7 @@ public class Section {
|
||||
for (String value : values) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
Set<Entity> found = findEntities(value.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
@@ -687,7 +576,7 @@ public class Section {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE);
|
||||
Set<Entity> found = findEntities(line.trim(), asType, false, true, ruleNumber, reason, legalBasis);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
|
||||
|
||||
if (redactEverywhere && !isLocal()) {
|
||||
@@ -731,13 +620,18 @@ public class Section {
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> findEntities(String value, String asType, boolean caseInsensitive, boolean redacted,
|
||||
int ruleNumber, String reason, String legalBasis, Engine engine) {
|
||||
private Set<Entity> findEntities(@Argument(ArgumentType.STRING) String value,
|
||||
@Argument(ArgumentType.TYPE) String asType,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean caseInsensitive,
|
||||
@Argument(ArgumentType.BOOLEAN) boolean redacted,
|
||||
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
|
||||
@Argument(ArgumentType.STRING) String reason,
|
||||
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
|
||||
|
||||
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
|
||||
String searchValue = caseInsensitive ? value.toLowerCase() : value;
|
||||
|
||||
Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, engine, false);
|
||||
Set<Entity> found = EntitySearchUtils.find(text, Set.of(searchValue), asType, headline, sectionNumber, false, false, Engine.RULE);
|
||||
|
||||
found.forEach(entity -> {
|
||||
if (redacted) {
|
||||
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Classification {
|
||||
|
||||
private Map<String, Float> probabilities = new HashMap<>();
|
||||
private String label;
|
||||
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class FilterGeometry {
|
||||
|
||||
private ImageSize imageSize;
|
||||
private ImageFormat imageFormat;
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Filters {
|
||||
|
||||
private FilterGeometry geometry;
|
||||
private Probability probability;
|
||||
private boolean allPassed;
|
||||
}
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Geometry {
|
||||
private float width;
|
||||
private float height;
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ImageFormat {
|
||||
|
||||
private float quotient;
|
||||
private boolean tooTall;
|
||||
private boolean tooWide;
|
||||
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ImageMetadata {
|
||||
|
||||
private Classification classification;
|
||||
private Position position;
|
||||
private Geometry geometry;
|
||||
private Filters filters;
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ImageServiceResponse {
|
||||
|
||||
private String dossierId;
|
||||
private String fileId;
|
||||
private List<ImageMetadata> imageMetadata = new ArrayList<>();
|
||||
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class ImageSize {
|
||||
|
||||
private float quotient;
|
||||
private boolean tooLarge;
|
||||
private boolean tooSmall;
|
||||
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Position {
|
||||
private float x1;
|
||||
private float x2;
|
||||
private float y1;
|
||||
private float y2;
|
||||
private int pageNumber;
|
||||
}
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.model.image;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
@Data
|
||||
public class Probability {
|
||||
|
||||
private boolean unconfident;
|
||||
|
||||
}
|
||||
+36
-39
@@ -1,34 +1,40 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
|
||||
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -47,7 +53,6 @@ public class AnalyzeService {
|
||||
private final SectionTextBuilderService sectionTextBuilderService;
|
||||
private final SectionGridCreatorService sectionGridCreatorService;
|
||||
private final NerAnalyserService nerAnalyserService;
|
||||
private final ImageService imageService;
|
||||
|
||||
|
||||
public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
|
||||
@@ -58,15 +63,9 @@ public class AnalyzeService {
|
||||
Document classifiedDoc;
|
||||
|
||||
try {
|
||||
|
||||
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest
|
||||
.getDossierId(), analyzeRequest.getFileId(), FileType.ORIGIN));
|
||||
|
||||
Map<Integer, List<PdfImage>> pdfImages = null;
|
||||
if (redactionServiceSettings.isEnableImageClassification()) {
|
||||
pdfImages = imageService.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream, pdfImages);
|
||||
classifiedDoc = pdfSegmentationService.parseDocument(storedObjectStream);
|
||||
pageCount = classifiedDoc.getPages().size();
|
||||
} catch (Exception e) {
|
||||
throw new RedactionException(e);
|
||||
@@ -75,7 +74,7 @@ public class AnalyzeService {
|
||||
List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
|
||||
sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
|
||||
|
||||
Text text = new Text(pageCount, sectionTexts);
|
||||
Text text = new Text(pageCount,sectionTexts);
|
||||
|
||||
// enhance section grid with headline data
|
||||
sectionTexts.forEach(sectionText -> classifiedDoc
|
||||
@@ -98,7 +97,7 @@ public class AnalyzeService {
|
||||
|
||||
var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
|
||||
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
|
||||
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
@@ -117,9 +116,10 @@ public class AnalyzeService {
|
||||
.getDossierTemplateId());
|
||||
|
||||
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(),
|
||||
redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
|
||||
rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), redactionLogEntries, legalBasis, dictionary
|
||||
.getVersion()
|
||||
.getDossierTemplateVersion(), dictionary.getVersion()
|
||||
.getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
|
||||
}
|
||||
@@ -149,7 +149,7 @@ public class AnalyzeService {
|
||||
}
|
||||
|
||||
var nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null) {
|
||||
if(redactionServiceSettings.isEnableEntityRecognition() && nerEntities == null){
|
||||
nerAnalyserService.computeNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||
}
|
||||
@@ -183,7 +183,7 @@ public class AnalyzeService {
|
||||
Set<Integer> sectionsToReanalyse = new HashSet<>();
|
||||
Map<Integer, Set<Image>> imageEntries = new HashMap<>();
|
||||
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
|
||||
if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
|
||||
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
|
||||
sectionsToReanalyse.add(entry.getSectionNumber());
|
||||
}
|
||||
if (entry.isImage()) {
|
||||
@@ -214,8 +214,7 @@ public class AnalyzeService {
|
||||
|
||||
excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
|
||||
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(),
|
||||
redactionLog, analyzeRequest.getAnalysisNumber());
|
||||
var redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog);
|
||||
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange
|
||||
.getRedactionLog());
|
||||
|
||||
@@ -228,13 +227,11 @@ public class AnalyzeService {
|
||||
.numberOfPages(text.getNumberOfPages())
|
||||
.hasUpdates(redactionLogChange.isHasChanges())
|
||||
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
|
||||
.analysisNumber(analyzeRequest.getAnalysisNumber())
|
||||
.rulesVersion(redactionLog.getRulesVersion())
|
||||
.dictionaryVersion(redactionLog.getDictionaryVersion())
|
||||
.legalBasisVersion(redactionLog.getLegalBasisVersion())
|
||||
.dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion())
|
||||
.wasReanalyzed(isReanalysis)
|
||||
.manualRedactions(analyzeRequest.getManualRedactions())
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -246,12 +243,12 @@ public class AnalyzeService {
|
||||
}
|
||||
|
||||
return Stream.concat(manualRedactions.getLegalBasisChanges()
|
||||
.stream()
|
||||
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
|
||||
.stream()
|
||||
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
|
||||
.stream()
|
||||
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
|
||||
.stream()
|
||||
.map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization()
|
||||
.stream()
|
||||
.map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove()
|
||||
.stream()
|
||||
.map(IdRemoval::getAnnotationId), manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))
|
||||
.collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
+7
-5
@@ -1,7 +1,6 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
@@ -16,10 +15,13 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
|
||||
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.*;
|
||||
import java.awt.Color;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.GregorianCalendar;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.*;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@@ -105,10 +107,10 @@ public class AnnotationService {
|
||||
|
||||
private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
|
||||
|
||||
if (redactionLogEntry.isLocalManualRedaction()) {
|
||||
if (redactionLogEntry.isManual()) {
|
||||
return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
|
||||
return "\nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry
|
||||
.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
|
||||
}
|
||||
|
||||
|
||||
+6
-6
@@ -6,6 +6,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
import feign.FeignException;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -13,8 +14,7 @@ import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.*;
|
||||
import java.util.List;
|
||||
import java.awt.Color;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@@ -63,7 +63,7 @@ public class DictionaryService {
|
||||
});
|
||||
});
|
||||
|
||||
if (dictionariesByDossier.containsKey(dossierId)) {
|
||||
if(dictionariesByDossier.containsKey(dossierId)) {
|
||||
dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary();
|
||||
dictionaryModels.forEach(dictionaryModel -> {
|
||||
dictionaryModel.getEntries().forEach(dictionaryEntry -> {
|
||||
@@ -105,7 +105,7 @@ public class DictionaryService {
|
||||
dictionaryRepresentation.setDictionaryVersion(version);
|
||||
dictionaryRepresentation.setDictionary(dictionary);
|
||||
|
||||
if (dossierId == null) {
|
||||
if(dossierId == null) {
|
||||
dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation);
|
||||
} else {
|
||||
dictionariesByDossier.put(dossierId, dictionaryRepresentation);
|
||||
@@ -122,7 +122,7 @@ public class DictionaryService {
|
||||
|
||||
dictionary.getDictionaryModels().forEach(dm -> {
|
||||
if (dm.isRecommendation() && !dm.getLocalEntries().isEmpty()) {
|
||||
dictionaryClient.addEntries(IdBuilder.getTypeId(dm.getType(), dossierTemplateId, dm.isDossierDictionary() ? dossierId : null), new ArrayList<>(dm.getLocalEntries()), false, true);
|
||||
dictionaryClient.addEntries(IdBuilder.getTypeId(dm.getType(), dossierTemplateId, dm.isDossierDictionary() ? dossierId : null), new ArrayList<>(dm.getLocalEntries()), false);
|
||||
long externalVersion = dictionaryClient.getVersion(dossierTemplateId);
|
||||
if (externalVersion == dictionary.getVersion().getDossierTemplateVersion() + 1) {
|
||||
dictionary.getVersion().setDossierTemplateVersion(externalVersion);
|
||||
@@ -204,7 +204,7 @@ public class DictionaryService {
|
||||
|
||||
//TODO merge dictionaries if they have same names
|
||||
long dossierDictionaryVersion = -1;
|
||||
if (dictionariesByDossier.containsKey(dossierId)) {
|
||||
if(dictionariesByDossier.containsKey(dossierId)) {
|
||||
var dossierRepresentation = dictionariesByDossier.get(dossierId);
|
||||
dossierRepresentation.getDictionary().forEach(dm -> {
|
||||
copy.add(SerializationUtils.clone(dm));
|
||||
|
||||
+65
-72
@@ -1,27 +1,40 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.kie.api.runtime.KieContainer;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -59,62 +72,38 @@ public class EntityRedactionService {
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
|
||||
for (SectionText reanalysisSection : reanalysisSections) {
|
||||
|
||||
Entities entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection
|
||||
Set<Entity> entities = findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection
|
||||
.getSectionNumber(), dictionary, local, nerEntities, reanalysisSection.getCellStarts());
|
||||
|
||||
if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
|
||||
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection
|
||||
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary, reanalysisSection
|
||||
.getCellStarts());
|
||||
} else {
|
||||
surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
|
||||
surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
|
||||
}
|
||||
|
||||
if (!local && analyzeRequest.getManualRedactions() != null) {
|
||||
|
||||
var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
|
||||
.filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
|
||||
.filter(fr -> fr.getRequestDate() != null)
|
||||
.collect(Collectors.toList());
|
||||
// only approved id removals, that haven't been forced back afterwards
|
||||
var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
|
||||
.filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
|
||||
.filter(idr -> idr.getRequestDate() != null)
|
||||
.filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
|
||||
.map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
|
||||
|
||||
if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
|
||||
.isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
|
||||
for (Image image : reanalysisSection.getImages()) {
|
||||
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
|
||||
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
|
||||
.getImageRecategorization()) {
|
||||
if (imageRecategorization.getStatus()
|
||||
.equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
|
||||
.equals(imageId)) {
|
||||
image.setType(imageRecategorization.getType());
|
||||
}
|
||||
}
|
||||
if (idsToRemove.contains(imageId)) {
|
||||
image.setIgnored(true);
|
||||
if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
|
||||
.isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
|
||||
.getImageRecategorization() != null) {
|
||||
for (Image image : reanalysisSection.getImages()) {
|
||||
String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
|
||||
for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
|
||||
.getImageRecategorization()) {
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
|
||||
.equals(imageId)) {
|
||||
image.setType(imageRecategorization.getType());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
|
||||
if (idsToRemove.contains(ps.getId())) {
|
||||
entity.setIgnored(true);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
|
||||
.isLocal(false)
|
||||
.dictionaryTypes(dictionary.getTypes())
|
||||
.entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream
|
||||
.concat(entities.getEntities().stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
|
||||
.concat(entities.stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber())
|
||||
.stream())
|
||||
.collect(Collectors.toSet()) : entities.getEntities())
|
||||
.nerEntities(entities.getNerEntities())
|
||||
.collect(Collectors.toSet()) : entities)
|
||||
.text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
|
||||
.searchText(reanalysisSection.getSearchableText().toString())
|
||||
.headline(reanalysisSection.getHeadline())
|
||||
@@ -160,8 +149,7 @@ public class EntityRedactionService {
|
||||
.add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
|
||||
.getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
|
||||
.getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
|
||||
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
|
||||
.getReferences()));
|
||||
.getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences()));
|
||||
}
|
||||
}
|
||||
return entitiesPerPage;
|
||||
@@ -207,46 +195,50 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
|
||||
private Entities findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local, NerEntities nerEntities,
|
||||
List<Integer> cellstarts) {
|
||||
private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber,
|
||||
Dictionary dictionary, boolean local, NerEntities nerEntities,
|
||||
List<Integer> cellstarts) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
String searchableString = searchableText.toString();
|
||||
if (StringUtils.isEmpty(searchableString)) {
|
||||
return new Entities(new HashSet<>(), new HashSet<>());
|
||||
return found;
|
||||
}
|
||||
|
||||
String lowercaseInputString = searchableString.toLowerCase();
|
||||
for (DictionaryModel model : dictionary.getDictionaryModels()) {
|
||||
if (model.isCaseInsensitive()) {
|
||||
EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(lowercaseInputString, model.getValues(local), model
|
||||
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false));
|
||||
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
|
||||
} else {
|
||||
EntitySearchUtils.addOrAddEngine(found, EntitySearchUtils.find(searchableString, model.getValues(local), model
|
||||
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY, false));
|
||||
.getType(), headline, sectionNumber, !local, model.isDossierDictionary(), local ? Engine.RULE : Engine.DICTIONARY));
|
||||
}
|
||||
}
|
||||
|
||||
Set<Entity> nerFound = new HashSet<>();
|
||||
if (!local) {
|
||||
nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellstarts, headline));
|
||||
Map<String, Set<String>> nerValuesPerType = getNerValues(sectionNumber, nerEntities, cellstarts);
|
||||
nerValuesPerType.entrySet().forEach(entry -> {
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(found, EntitySearchUtils.find(searchableString, entry.getValue(), entry
|
||||
.getKey(), headline, sectionNumber, false, false, Engine.NER), dictionary);
|
||||
});
|
||||
}
|
||||
|
||||
return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound);
|
||||
return EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary);
|
||||
}
|
||||
|
||||
|
||||
private Set<Entity> getNerValues(int sectionNumber, NerEntities nerEntities,
|
||||
List<Integer> cellstarts, String headline) {
|
||||
private Map<String, Set<String>> getNerValues(int sectionNumber, NerEntities nerEntities,
|
||||
List<Integer> cellstarts) {
|
||||
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
Map<String, Set<String>> nerValuesPerType = new HashMap<>();
|
||||
|
||||
if (redactionServiceSettings.isEnableEntityRecognition() && nerEntities.getResult()
|
||||
.containsKey(sectionNumber)) {
|
||||
nerEntities.getResult().get(sectionNumber).forEach(res -> {
|
||||
if (cellstarts == null || cellstarts.isEmpty()) {
|
||||
entities.add(new Entity(new String(Base64.decodeBase64(res.getValue().getBytes())), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
|
||||
.add(new String(Base64.decodeBase64(res.getValue().getBytes())));
|
||||
} else {
|
||||
boolean intersectsCellStart = false;
|
||||
for (Integer cellStart : cellstarts) {
|
||||
@@ -255,12 +247,13 @@ public class EntityRedactionService {
|
||||
}
|
||||
}
|
||||
if (!intersectsCellStart) {
|
||||
entities.add(new Entity(new String(Base64.decodeBase64(res.getValue().getBytes())), res.getType(), res.getStartOffset(), res.getEndOffset(), headline, sectionNumber, false, false, Engine.NER));
|
||||
nerValuesPerType.computeIfAbsent(res.getType(), (a) -> new HashSet<>())
|
||||
.add(new String(Base64.decodeBase64(res.getValue().getBytes())));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return entities;
|
||||
return nerValuesPerType;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.client.MockMultipartFile;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageClassificationService {
|
||||
|
||||
private final ImageClassificationClient imageClassificationClient;
|
||||
private final RedactionServiceSettings settings;
|
||||
|
||||
|
||||
public void classifyImages(Page page) {
|
||||
|
||||
page.getImages().forEach(image -> {
|
||||
|
||||
if (settings.isEnableImageClassification() && !isEntirePageImage(image, page)) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ImageIO.write(image.getImage(), "png", baos);
|
||||
var mockFile = new MockMultipartFile("file", "Image.png", "image/png", baos.toByteArray());
|
||||
ImageClassificationResponse response = imageClassificationClient.classify(mockFile);
|
||||
image.setImageType(ImageType.valueOf(response.getCategory()));
|
||||
} catch (Exception e) {
|
||||
log.error("Could not classify image", e);
|
||||
image.setImageType(ImageType.OTHER);
|
||||
}
|
||||
log.info("Image classification took: " + (System.currentTimeMillis() - start));
|
||||
} else {
|
||||
image.setImageType(ImageType.OTHER);
|
||||
}
|
||||
|
||||
image.getImage().flush();
|
||||
image.setImage(null);
|
||||
|
||||
if (image.getImageType().equals(ImageType.OTHER)) {
|
||||
page.getTextBlocks().forEach(textblock -> {
|
||||
if (image.getPosition()
|
||||
.contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
|
||||
image.setImageType(ImageType.OCR);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private boolean isEntirePageImage(PdfImage image, Page page){
|
||||
double imageArea = image.getPosition().getHeight() * image.getPosition().getWidth();
|
||||
if(imageArea / page.getCropBoxArea() >= settings.getMaxImageCropboxRatio()){
|
||||
log.info("Skipping image classification because images is almost as large as the entire page");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
+11
-16
@@ -1,9 +1,16 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
@@ -13,16 +20,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ManualRedactionSurroundingTextService {
|
||||
@@ -83,16 +83,11 @@ public class ManualRedactionSurroundingTextService {
|
||||
List<Rectangle> toFindPositions) {
|
||||
|
||||
Set<Entity> entities = EntitySearchUtils.find(sectionText.getText(), Set.of(value), "dummy", sectionText.getHeadline(), sectionText
|
||||
.getSectionNumber(), false, false, Engine.DICTIONARY, false);
|
||||
.getSectionNumber(), false, false, Engine.DICTIONARY);
|
||||
Set<Entity> entitiesWithPositions = EntitySearchUtils.clearAndFindPositions(entities, sectionText.getSearchableText(), null);
|
||||
|
||||
Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions);
|
||||
|
||||
if (correctEntity == null) {
|
||||
log.warn("Could not calculate surrounding text");
|
||||
return Pair.of("", "");
|
||||
}
|
||||
|
||||
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
|
||||
surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText
|
||||
.getCellStarts());
|
||||
|
||||
+32
-20
@@ -1,14 +1,25 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.redaction.v1.model.Change;
|
||||
import com.iqser.red.service.redaction.v1.model.ChangeType;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogChanges;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -18,7 +29,7 @@ public class RedactionChangeLogService {
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
|
||||
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
|
||||
public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
|
||||
@@ -26,23 +37,24 @@ public class RedactionChangeLogService {
|
||||
|
||||
if (previousRedactionLog == null) {
|
||||
currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
|
||||
entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
|
||||
entry.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
});
|
||||
return new RedactionLogChanges(currentRedactionLog, false);
|
||||
}
|
||||
|
||||
List<RedactionLogEntry> notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> !entry.lastChangeIsRemoved())
|
||||
.filter(entry -> !entry.getChanges()
|
||||
.get(entry.getChanges().size() - 1)
|
||||
.getType()
|
||||
.equals(ChangeType.REMOVED))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
Set<RedactionLogEntry> added = currentRedactionLog.getRedactionLogEntry().stream()
|
||||
.filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()).collect(Collectors.toSet());
|
||||
|
||||
notRemovedPreviousEntries.forEach(added::remove);
|
||||
Set<RedactionLogEntry> added = new HashSet<>(currentRedactionLog.getRedactionLogEntry());
|
||||
added.removeAll(notRemovedPreviousEntries);
|
||||
|
||||
Set<RedactionLogEntry> removed = new HashSet<>(notRemovedPreviousEntries);
|
||||
currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
|
||||
removed.removeAll(currentRedactionLog.getRedactionLogEntry());
|
||||
|
||||
Map<String, RedactionLogEntry> addedIds = new HashMap<>();
|
||||
added.forEach(entry -> {
|
||||
@@ -60,16 +72,16 @@ public class RedactionChangeLogService {
|
||||
newRedactionLogEntries.forEach(entry -> {
|
||||
if (removedIds.contains(entry.getId()) && addedIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
|
||||
changes.add(new Change(ChangeType.CHANGED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
toRemove.add(entry);
|
||||
} else if (removedIds.contains(entry.getId())) {
|
||||
entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
|
||||
entry.getChanges().add(new Change(ChangeType.REMOVED, OffsetDateTime.now()));
|
||||
} else if (addedIds.containsKey(entry.getId())) {
|
||||
List<Change> changes = entry.getChanges();
|
||||
changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
|
||||
changes.add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
var newEntry = addedIds.get(entry.getId());
|
||||
newEntry.setChanges(changes);
|
||||
addedIds.put(entry.getId(), newEntry);
|
||||
@@ -80,8 +92,8 @@ public class RedactionChangeLogService {
|
||||
newRedactionLogEntries.removeAll(toRemove);
|
||||
|
||||
addedIds.forEach((k, v) -> {
|
||||
if (v.getChanges().isEmpty()) {
|
||||
v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
|
||||
if(v.getChanges().isEmpty()) {
|
||||
v.getChanges().add(new Change(ChangeType.ADDED, OffsetDateTime.now()));
|
||||
}
|
||||
newRedactionLogEntries.add(v);
|
||||
});
|
||||
|
||||
+23
-9
@@ -1,5 +1,15 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
@@ -10,13 +20,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
@Slf4j
|
||||
@@ -64,6 +70,7 @@ public class RedactionLogCreatorService {
|
||||
.legalBasis(image.getLegalBasis())
|
||||
.matchedRule(image.getMatchedRule())
|
||||
.isHint(dictionaryService.isHint(image.getType(), dossierTemplateId))
|
||||
.manual(false)
|
||||
.isDictionaryEntry(false)
|
||||
.isRecommendation(false)
|
||||
.positions(List.of(new Rectangle(new Point((float) image.getPosition()
|
||||
@@ -133,14 +140,14 @@ public class RedactionLogCreatorService {
|
||||
} else {
|
||||
float x = textPositions.get(0).getXDirAdj();
|
||||
float y = textPositions.get(0).getYDirAdj();
|
||||
float width = textPositions.get(0).getWidthDirAdj();
|
||||
float width = textPositions.get(0).getWidth();
|
||||
float height = textPositions.get(0).getHeightDir();
|
||||
int startIndex = 0;
|
||||
|
||||
for (int i = 1; i < textPositions.size(); i++) {
|
||||
float xDirAdj = textPositions.get(i).getXDirAdj();
|
||||
float yDirAdj = textPositions.get(i).getYDirAdj();
|
||||
float widthDir = textPositions.get(i).getWidthDirAdj();
|
||||
float widthDir = textPositions.get(i).getWidth();
|
||||
float heightDir = textPositions.get(i).getHeightDir();
|
||||
|
||||
if (!(isCharInSameLine(y, yDirAdj, height, heightDir) && isCharClose(x, xDirAdj, width))) {
|
||||
@@ -166,7 +173,10 @@ public class RedactionLogCreatorService {
|
||||
private boolean isCharClose(float x, float xDirAdj, float width) {
|
||||
|
||||
float max = x + (5 * width);
|
||||
return xDirAdj < max;
|
||||
if (xDirAdj < max) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -180,7 +190,11 @@ public class RedactionLogCreatorService {
|
||||
float minY = y - offsetY;
|
||||
float maxY = y + offsetY;
|
||||
|
||||
return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight;
|
||||
if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+87
-65
@@ -1,19 +1,36 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import com.iqser.red.service.redaction.v1.model.*;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
|
||||
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionGrid;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -48,7 +65,7 @@ public class RedactionLogMergeService {
|
||||
|
||||
if (excludedPages != null && !excludedPages.isEmpty()) {
|
||||
entry.getPositions().forEach(pos -> {
|
||||
if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) {
|
||||
if (!entry.isManual() && excludedPages.contains(pos.getPage())) {
|
||||
entry.setExcluded(true);
|
||||
}
|
||||
});
|
||||
@@ -119,9 +136,9 @@ public class RedactionLogMergeService {
|
||||
var imageRecategorization = (ManualImageRecategorization) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
|
||||
|
||||
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
redactionLogEntry.setType(imageRecategorization.getType());
|
||||
redactionLogEntry.setHasBeenRecategorized(true);
|
||||
redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
|
||||
|
||||
if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
|
||||
@@ -134,65 +151,67 @@ public class RedactionLogMergeService {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
|
||||
} else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
|
||||
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
|
||||
redactionLogEntry.setRecategorizationType(imageRecategorization.getType());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
|
||||
}
|
||||
|
||||
redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization)
|
||||
.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
|
||||
.withChange("type", imageRecategorization.getType()));
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof IdRemoval) {
|
||||
var manualRemoval = (IdRemoval) mrw.getItem();
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) {
|
||||
log.debug("Skipping merge for dictionary-modifying entry");
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
|
||||
redactionLogEntry.setHasBeenRemovedByManualOverride(true);
|
||||
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
|
||||
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
|
||||
} else {
|
||||
String manualOverrideReason = null;
|
||||
if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setRedacted(false);
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
|
||||
redactionLogEntry.setHint(false);
|
||||
} else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval)
|
||||
.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ?
|
||||
ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
|
||||
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
|
||||
}
|
||||
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser());
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
|
||||
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
|
||||
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof ManualForceRedaction) {
|
||||
var manualForceRedact = (ManualForceRedaction) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
|
||||
if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
// Forcing a skipped hint should result in a hint
|
||||
if (dictionaryIsHint) {
|
||||
redactionLogEntry.setHint(true);
|
||||
} else {
|
||||
redactionLogEntry.setRedacted(true);
|
||||
}
|
||||
redactionLogEntry.setRedacted(true);
|
||||
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
|
||||
redactionLogEntry.setHasBeenForced(true);
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
|
||||
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
|
||||
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
|
||||
}
|
||||
|
||||
redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
var manualChange = ManualChange.from(manualForceRedact)
|
||||
.withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
|
||||
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
|
||||
|
||||
}
|
||||
|
||||
@@ -200,9 +219,11 @@ public class RedactionLogMergeService {
|
||||
var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
|
||||
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
|
||||
redactionLogEntry.setRedacted(true);
|
||||
redactionLogEntry.setHasLegalBasisChanged(true);
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
redactionLogEntry.setSection(manualLegalBasisChange.getSection());
|
||||
}
|
||||
@@ -211,45 +232,47 @@ public class RedactionLogMergeService {
|
||||
}
|
||||
} else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
|
||||
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
|
||||
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
|
||||
}
|
||||
|
||||
redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
|
||||
var manualChange = ManualChange.from(manualLegalBasisChange)
|
||||
.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
|
||||
manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
|
||||
if (manualLegalBasisChange.getSection() != null) {
|
||||
manualChange.withChange("section", manualLegalBasisChange.getSection());
|
||||
}
|
||||
if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
|
||||
manualChange.withChange("value", manualLegalBasisChange.getValue());
|
||||
}
|
||||
redactionLogEntry.getManualChanges().add(manualChange);
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
|
||||
}
|
||||
|
||||
if (mrw.getItem() instanceof ManualResizeRedaction) {
|
||||
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
|
||||
String manualOverrideReason = null;
|
||||
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
redactionLogEntry.setValue(manualResizeRedact.getValue());
|
||||
redactionLogEntry.setHasBeenResized(true);
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
|
||||
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
|
||||
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
|
||||
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
|
||||
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
|
||||
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
|
||||
redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
|
||||
redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
|
||||
} else {
|
||||
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
|
||||
}
|
||||
|
||||
redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
|
||||
redactionLogEntry.setReason(manualOverrideReason);
|
||||
redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact)
|
||||
.withManualRedactionType(ManualRedactionType.RESIZE)
|
||||
.withChange("value", manualResizeRedact.getValue()));
|
||||
redactionLogEntry.setManual(true);
|
||||
redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE);
|
||||
|
||||
}
|
||||
|
||||
});
|
||||
@@ -305,17 +328,14 @@ public class RedactionLogMergeService {
|
||||
|
||||
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
|
||||
|
||||
return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) &&
|
||||
manualRedactionEntry.getProcessedDate() != null &&
|
||||
(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
|
||||
return manualRedactionEntry.getStatus()
|
||||
.equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
|
||||
}
|
||||
|
||||
|
||||
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
|
||||
String dossierTemplateId) {
|
||||
|
||||
var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
|
||||
|
||||
return RedactionLogEntry.builder()
|
||||
.id(id)
|
||||
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
|
||||
@@ -329,9 +349,11 @@ public class RedactionLogMergeService {
|
||||
.redacted(true)
|
||||
.isHint(false)
|
||||
.sectionNumber(-1)
|
||||
.manual(true)
|
||||
.status(manualRedactionEntry.getStatus())
|
||||
.manualRedactionType(ManualRedactionType.ADD)
|
||||
.manualRedactionUserId(manualRedactionEntry.getUser())
|
||||
.rectangle(manualRedactionEntry.isRectangle())
|
||||
.manualChanges(List.of(ManualChange.from(manualRedactionEntry)
|
||||
.withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
+6
-4
@@ -1,5 +1,10 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.CellRectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
|
||||
@@ -9,11 +14,8 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
|
||||
+27
-11
@@ -1,19 +1,35 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Collectors;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.SectionArea;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Header;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ public class EntitySearchUtils {
|
||||
|
||||
|
||||
public Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber,
|
||||
boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine, boolean ignoreMinLength) {
|
||||
boolean isDictionaryEntry, boolean isDossierDictionary, Engine engine) {
|
||||
|
||||
Set<Entity> found = new HashSet<>();
|
||||
|
||||
@@ -63,7 +63,7 @@ public class EntitySearchUtils {
|
||||
|
||||
String cleanValue = value.trim();
|
||||
|
||||
if (!ignoreMinLength && cleanValue.length() <= 2) {
|
||||
if (cleanValue.length() <= 2) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import java.awt.Graphics;
|
||||
import java.awt.geom.Rectangle2D;
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageMergeService {
|
||||
|
||||
|
||||
public List<PdfImage> mergeImages(List<PdfImage> images, int rotation){
|
||||
|
||||
List<PdfImage> mergedList = processImages(images, rotation);
|
||||
|
||||
List<PdfImage> imagesInImage = new ArrayList<>();
|
||||
for(PdfImage image: mergedList){
|
||||
for (PdfImage inner: mergedList){
|
||||
if(image != inner && image.getPosition().contains(inner.getPosition().getX(), inner.getPosition().getY(), inner.getPosition().getWidth(), inner.getPosition().getHeight())){
|
||||
imagesInImage.add(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
mergedList.removeAll(imagesInImage);
|
||||
|
||||
return mergedList;
|
||||
}
|
||||
|
||||
|
||||
//merge images, if they are separated during pdf import, return new list of Pdfimages
|
||||
private List<PdfImage> processImages(List<PdfImage> imageList, int rotation) {
|
||||
if (imageList.size() > 1) {
|
||||
List<PdfImage> mergedList = new ArrayList<>();
|
||||
int countElementsInList = 0;
|
||||
boolean beginImage = true;
|
||||
|
||||
// a List of Boolean, true = candidate for merging, false = no merging
|
||||
List<Boolean> candidatesList = getCandidatesList(imageList, rotation);
|
||||
|
||||
// loop through list, if there are candidates for merging (true), merge images and add it to mergedList
|
||||
for (int i = 0; i < candidatesList.size(); i++) {
|
||||
if (candidatesList.get(i)) {
|
||||
if (beginImage) {
|
||||
//begin of image, merge two parts of imageList
|
||||
PdfImage mergedImage = mergeTwoImages(imageList.get(i), imageList.get(i + 1), rotation);
|
||||
// image merge successful
|
||||
if (mergedImage != null) {
|
||||
mergedList.add(mergedImage);
|
||||
countElementsInList++;
|
||||
}
|
||||
} else {
|
||||
//middle of an image, merge current piece auf mergedList with image of imageList
|
||||
PdfImage mergedImage = mergeTwoImages(mergedList.get(countElementsInList - 1), imageList.get(i + 1), rotation);
|
||||
// image merge successful
|
||||
if (mergedImage != null) {
|
||||
mergedList.set(countElementsInList - 1, mergedImage);
|
||||
}
|
||||
}
|
||||
beginImage = false;
|
||||
} else {
|
||||
// if the last candidate is false, then both images i and i+1 must be added
|
||||
if (i == candidatesList.size() - 1) {
|
||||
if (countElementsInList > 0 && mergedList.get(countElementsInList - 1) == imageList.get(i)) {
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
} else {
|
||||
mergedList.add(imageList.get(i));
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
}
|
||||
} else {
|
||||
//first image is not splitted, add i to resultlist
|
||||
if (beginImage) {
|
||||
mergedList.add(imageList.get(i));
|
||||
countElementsInList++;
|
||||
} else {
|
||||
// i is the end of an image, add begin of new image
|
||||
mergedList.add(imageList.get(i + 1));
|
||||
countElementsInList++;
|
||||
beginImage = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return mergedList;
|
||||
} else {
|
||||
return imageList;
|
||||
}
|
||||
}
|
||||
|
||||
private PdfImage mergeTwoImages(PdfImage image1, PdfImage image2, int rotation) {
|
||||
|
||||
// diese Angaben von getPosition scheinen nicht richtig zu sein, damit werden teile des Bildes abgeschnitten
|
||||
double width = image1.getPosition().getWidth();
|
||||
double width2 = image2.getPosition().getWidth();
|
||||
double height1 = image1.getPosition().getHeight();
|
||||
double height2 = image2.getPosition().getHeight();
|
||||
// mit den Werten, die unter Image gespeichert sind, funktioniert es
|
||||
double img1height = image1.getImage().getHeight();
|
||||
double img1width = image1.getImage().getWidth();
|
||||
double img2height = image2.getImage().getHeight();
|
||||
|
||||
BufferedImage mergedImage = new BufferedImage((int) img1width, (int) (img1height + img2height), BufferedImage.TYPE_INT_RGB);
|
||||
Graphics mergedImageGraphics = mergedImage.getGraphics();
|
||||
try {
|
||||
mergedImageGraphics.drawImage(image1.getImage(), 0, 0, null);
|
||||
mergedImageGraphics.drawImage(image2.getImage(), 0, (int) (img1height), null);
|
||||
|
||||
// set Image, Position and type for merged Image
|
||||
//set position for merged image with values of image1 and the height of both
|
||||
Rectangle2D pos = new Rectangle2D.Float();
|
||||
pos.setRect(image1.getPosition().getX(), image2.getPosition().getY(), rotation == 90 ? width + width2: width, rotation == 90 ? height1 : height1 + height2);
|
||||
PdfImage newPdfImage = new PdfImage(mergedImage, pos, image1.getPage(), image1.isHasTransparency() || image2.isHasTransparency());
|
||||
// Graphics need to be disposed
|
||||
|
||||
image1.getImage().flush();
|
||||
image2.getImage().flush();
|
||||
|
||||
mergedImage.flush();
|
||||
mergedImageGraphics.dispose();
|
||||
|
||||
return newPdfImage;
|
||||
} catch (Exception e) {
|
||||
// failed to merge image
|
||||
log.error("Failed to merge image", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
//make a list of true and false, if the image is a candidate for merging
|
||||
private List<Boolean> getCandidatesList(List<PdfImage> imageList, int rotation) {
|
||||
List<Boolean> candidatesList = new ArrayList<>();
|
||||
for (int i = 0; i < imageList.size(); i++) {
|
||||
if (i >= 1) {
|
||||
candidatesList.add(isCandidateForMerging(imageList.get(i - 1), imageList.get(i), rotation));
|
||||
}
|
||||
}
|
||||
return candidatesList;
|
||||
}
|
||||
|
||||
// evaluate if two images are candidates for merging, depending on their coordinates, width and height
|
||||
private boolean isCandidateForMerging(PdfImage image1, PdfImage image2, int rotation) {
|
||||
double x1 = rotation == 90 ? image1.getPosition().getY() : image1.getPosition().getX();
|
||||
double y1 = rotation == 90 ? image1.getPosition().getX() : image1.getPosition().getY();
|
||||
double width1 = rotation == 90 ? image1.getPosition().getHeight() : image1.getPosition().getWidth();
|
||||
double x2 = rotation == 90 ? image2.getPosition().getY() : image2.getPosition().getX();
|
||||
double y2 = rotation == 90 ? image2.getPosition().getX() : image2.getPosition().getY();
|
||||
double width2 = rotation == 90 ? image2.getPosition().getHeight() : image2.getPosition().getWidth();
|
||||
double height2 = rotation == 90 ? image2.getPosition().getWidth() : image2.getPosition().getHeight();
|
||||
//if the x-coordinates and widths of images are equal and the height is equal to difference between y-coordinates,
|
||||
// then it is the same picture and has to be merged -> return true
|
||||
return x1 == x2 && width1 == width2 && Math.ceil(height2) == Math.ceil(rotation == 90 ? y2 - y1 : y1 - y2) && width2 > (height2 / 6);
|
||||
}
|
||||
|
||||
}
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageService {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RedactionStorageService redactionStorageService;
|
||||
|
||||
@SneakyThrows
|
||||
public Map<Integer, List<PdfImage>> convertImages(String dossierId, String fileId){
|
||||
|
||||
var imageClassificationStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
|
||||
ImageServiceResponse imageServiceResponse = objectMapper.readValue(imageClassificationStream, ImageServiceResponse.class);
|
||||
|
||||
Map<Integer, List<PdfImage>> images = new HashMap<>();
|
||||
imageServiceResponse.getImageMetadata().stream().forEach(imageMetadata -> {
|
||||
var classification = imageMetadata.getFilters().isAllPassed() ? ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)) : ImageType.OTHER;
|
||||
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), classification, imageMetadata.getPosition().getPageNumber()));
|
||||
});
|
||||
|
||||
return images;
|
||||
}
|
||||
|
||||
|
||||
public void findOcr(Page page){
|
||||
page.getImages().forEach(image -> {
|
||||
if (image.getImageType().equals(ImageType.OTHER)) {
|
||||
page.getTextBlocks().forEach(textblock -> {
|
||||
if (image.getPosition()
|
||||
.contains(textblock.getMinX(), textblock.getMinY(), textblock.getWidth(), textblock.getHeight())) {
|
||||
image.setImageType(ImageType.OCR);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+14
-6
@@ -10,7 +10,6 @@ import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
@@ -30,6 +29,7 @@ import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ImageClassificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
@@ -50,10 +50,17 @@ public class PdfSegmentationService {
|
||||
private final BlockificationService blockificationService;
|
||||
private final ClassificationService classificationService;
|
||||
private final SectionsBuilderService sectionsBuilderService;
|
||||
private final ImageService imageService;
|
||||
private final ImageClassificationService imageClassificationService;
|
||||
private final ImageMergeService imageMergeService;
|
||||
|
||||
|
||||
public Document parseDocument(InputStream documentInputStream, Map<Integer, List<PdfImage>> pdfImages) throws IOException {
|
||||
public Document parseDocument(InputStream documentInputStream) throws IOException {
|
||||
|
||||
return parseDocument(documentInputStream, false);
|
||||
}
|
||||
|
||||
|
||||
public Document parseDocument(InputStream documentInputStream, boolean ignoreImages) throws IOException {
|
||||
|
||||
PDDocument pdDocument = null;
|
||||
try {
|
||||
@@ -115,14 +122,15 @@ public class PdfSegmentationService {
|
||||
page.setLandscape(isLandscape || isRotated);
|
||||
page.setPageNumber(pageNumber);
|
||||
|
||||
List<PdfImage> mergedList = imageMergeService.mergeImages(stripper.getImages(), rotation);
|
||||
page.setImages(mergedList);
|
||||
|
||||
tableExtractionService.extractTables(cleanRulings, page);
|
||||
buildPageStatistics(page);
|
||||
increaseDocumentStatistics(page, document);
|
||||
|
||||
if (pdfImages != null && pdfImages.containsKey(pageNumber)) {
|
||||
page.setImages(pdfImages.get(pageNumber));
|
||||
imageService.findOcr(page);
|
||||
if (!ignoreImages) {
|
||||
imageClassificationService.classifyImages(page);
|
||||
}
|
||||
|
||||
pages.add(page);
|
||||
|
||||
+1
-5
@@ -211,11 +211,7 @@ public class SectionsBuilderService {
|
||||
if (previous != null && previous.getText().startsWith("Table ")) {
|
||||
table.setHeadline(previous.getText());
|
||||
} else {
|
||||
if(lastHeadline == null || lastHeadline.isEmpty()) {
|
||||
table.setHeadline("Text in table");
|
||||
} else {
|
||||
table.setHeadline("Table in: " + lastHeadline);
|
||||
}
|
||||
table.setHeadline("Table in: " + lastHeadline);
|
||||
}
|
||||
|
||||
if (textBlock != null && !alreadyAdded) {
|
||||
|
||||
-2
@@ -127,8 +127,6 @@ public class RedactionStorageService {
|
||||
return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+41
-121
@@ -23,8 +23,8 @@ import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
@@ -52,7 +52,13 @@ import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
|
||||
@@ -70,13 +76,13 @@ import com.iqser.red.service.redaction.v1.model.RedactionResult;
|
||||
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
|
||||
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.ImageClassificationClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.client.RulesClient;
|
||||
import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
|
||||
import com.iqser.red.service.redaction.v1.server.memory.MemoryStats;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
|
||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||
@@ -131,6 +137,9 @@ public class RedactionIntegrationTest {
|
||||
@MockBean
|
||||
private DictionaryClient dictionaryClient;
|
||||
|
||||
@MockBean
|
||||
private ImageClassificationClient imageClassificationClient;
|
||||
|
||||
@Autowired
|
||||
private RedactionStorageService redactionStorageService;
|
||||
|
||||
@@ -586,7 +595,7 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated3.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated3.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long rstart = System.currentTimeMillis();
|
||||
@@ -613,65 +622,12 @@ public class RedactionIntegrationTest {
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var values = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.map(RedactionLogEntry::getValue)
|
||||
.collect(Collectors.toList());
|
||||
var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
|
||||
|
||||
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testIgnoreHint() {
|
||||
|
||||
System.out.println("testIgnoreHint");
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/new/test-ignore-hint.pdf");
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var toRemove = IdRemoval.builder()
|
||||
.annotationId("c630599611e6e3db314518374bcf70f7")
|
||||
.status(AnnotationStatus.APPROVED)
|
||||
.user("test")
|
||||
.removeFromDictionary(false)
|
||||
.processedDate(OffsetDateTime.now())
|
||||
.requestDate(OffsetDateTime.now())
|
||||
.build();
|
||||
|
||||
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(toRemove)).build();
|
||||
request.setManualRedactions(manualRedactions);
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder()
|
||||
.manualRedactions(manualRedactions)
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||
.findAny()
|
||||
.get();
|
||||
assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
|
||||
|
||||
var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||
.findAny()
|
||||
.get();
|
||||
assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@Ignore
|
||||
public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
|
||||
@@ -729,7 +685,7 @@ public class RedactionIntegrationTest {
|
||||
public void redactionTestSeparatedRedaction() throws IOException {
|
||||
|
||||
String fileName = "scanned/VV-380943_page38.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
|
||||
String outputFileName = getTemporaryDirectory() + "/AnnotatedRedactionTestSeparatedRedaction.pdf";
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
@@ -753,7 +709,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
System.out.println("first analysis duration: " + (end - start));
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) {
|
||||
fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
|
||||
}
|
||||
|
||||
@@ -838,60 +794,15 @@ public class RedactionIntegrationTest {
|
||||
|
||||
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
System.out.println("Output file:" + outputFileName);
|
||||
System.out.println("hi");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChangeComputation() throws IOException {
|
||||
|
||||
String fileName = "files/new/test1S1T1.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
analyzeService.analyze(request);
|
||||
|
||||
dictionary.get(AUTHOR).add("report");
|
||||
reanlysisVersions.put("report", 2L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
dictionary.get(AUTHOR).add("assessment report");
|
||||
reanlysisVersions.put("assessment report", 3L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var changes = redactionLog.getRedactionLogEntry().stream().filter(entry ->
|
||||
entry.getValue() != null && entry.getValue().equals("report"))
|
||||
.findFirst().get().getChanges();
|
||||
|
||||
assertThat(changes.size()).isEqualTo(2);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
String fileName = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
String fileName = "files/new/VV-919901.pdf";
|
||||
String outputFileName = getTemporaryDirectory() + "/Annotated.pdf";
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ClassPathResource pdfFileResource = new ClassPathResource(fileName);
|
||||
@@ -915,7 +826,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
System.out.println("first analysis duration: " + (end - start));
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Test.json")) {
|
||||
fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID)));
|
||||
}
|
||||
|
||||
@@ -1000,7 +911,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
System.out.println("Output file:" + outputFileName);
|
||||
System.out.println("hi");
|
||||
}
|
||||
|
||||
|
||||
@@ -1019,7 +930,7 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
@@ -1113,7 +1024,7 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
@@ -1139,7 +1050,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
RedactionResult result = redactionController.classify(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Classified.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
@@ -1161,7 +1072,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
RedactionResult result = redactionController.sections(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Sections.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Sections.pdf")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
@@ -1183,7 +1094,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
@@ -1205,7 +1116,7 @@ public class RedactionIntegrationTest {
|
||||
|
||||
RedactionResult result = redactionController.htmlTables(redactionRequest);
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Tables.html")) {
|
||||
fileOutputStream.write(result.getDocument());
|
||||
}
|
||||
}
|
||||
@@ -1275,7 +1186,7 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
@@ -1408,13 +1319,12 @@ public class RedactionIntegrationTest {
|
||||
.manualRedactions(manualRedactions)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testExpandByRegEx() throws IOException {
|
||||
|
||||
@@ -1430,7 +1340,7 @@ public class RedactionIntegrationTest {
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
@@ -1508,7 +1418,7 @@ public class RedactionIntegrationTest {
|
||||
.manualRedactions(manualRedactions)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
|
||||
@@ -1556,4 +1466,14 @@ public class RedactionIntegrationTest {
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static String getTemporaryDirectory() {
|
||||
|
||||
String tmpdir = System.getProperty("java.io.tmpdir");
|
||||
if (StringUtils.isNotBlank(tmpdir)) {
|
||||
return tmpdir;
|
||||
}
|
||||
return "/tmp";
|
||||
}
|
||||
|
||||
}
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.utils;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class OsUtils {
|
||||
|
||||
public static boolean isWindows() {
|
||||
|
||||
return StringUtils.containsIgnoreCase(System.getProperty("os.name"), "Windows");
|
||||
}
|
||||
|
||||
|
||||
public static String getTemporaryDirectory() {
|
||||
|
||||
String tmpdir = System.getProperty("java.io.tmpdir");
|
||||
if (isWindows() && StringUtils.isNotBlank(tmpdir)) {
|
||||
return tmpdir;
|
||||
}
|
||||
return "/tmp";
|
||||
}
|
||||
|
||||
}
|
||||
+37
-27
@@ -1,16 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.segmentation;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.iqser.red.service.redaction.v1.server.Application;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
|
||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.image.ImageServiceResponse;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
@@ -35,18 +31,12 @@ import javax.imageio.ImageIO;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils.getTemporaryDirectory;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import lombok.SneakyThrows;
|
||||
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@@ -77,29 +67,42 @@ public class PdfSegmentationServiceTest {
|
||||
@MockBean
|
||||
private LegalBasisClient legalBasisClient;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Configuration
|
||||
@EnableAutoConfiguration(exclude = { RabbitAutoConfiguration.class})
|
||||
public static class TestConfiguration {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMergeImages() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/270Rotated.pdf");
|
||||
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
assertThat(document.getPages().get(0).getImages().size()).isEqualTo(1);
|
||||
assertThat(document.getPages().get(1).getImages().size()).isEqualTo(0);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
@SneakyThrows
|
||||
public void testMapping(){
|
||||
ClassPathResource responseJson = new ClassPathResource("files/image_response.json");
|
||||
ImageServiceResponse imageServiceResponse = objectMapper.readValue(responseJson.getInputStream(), ImageServiceResponse.class);
|
||||
@Ignore
|
||||
public void testExtractImages() throws IOException {
|
||||
|
||||
Map<Integer, List<PdfImage>> images = new HashMap<>();
|
||||
imageServiceResponse.getImageMetadata().stream().forEach(imageMetadata -> {
|
||||
images.computeIfAbsent(imageMetadata.getPosition().getPageNumber() ,x -> new ArrayList<>())
|
||||
.add(new PdfImage(new RedRectangle2D(imageMetadata.getPosition().getX1(), imageMetadata.getPosition().getY1(), imageMetadata.getGeometry().getWidth(), imageMetadata.getGeometry().getHeight()), ImageType.valueOf(imageMetadata.getClassification().getLabel().toUpperCase(Locale.ROOT)), imageMetadata.getPosition().getPageNumber()));
|
||||
});
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/new/Single Study - Oral (Gavage) Mouse.pdf");
|
||||
|
||||
System.out.println("object");
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
int i = 0;
|
||||
for (Page page : document.getPages()) {
|
||||
for (PdfImage image : page.getImages()) {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ImageIO.write(image.getImage(), "png", baos);
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Image " + i + ".png")) {
|
||||
fileOutputStream.write(baos.toByteArray());
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +111,7 @@ public class PdfSegmentationServiceTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
|
||||
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream(), null);
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
@@ -129,7 +132,7 @@ public class PdfSegmentationServiceTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
|
||||
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream(), null);
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
@@ -167,7 +170,7 @@ public class PdfSegmentationServiceTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
|
||||
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream(), null);
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
@@ -205,7 +208,7 @@ public class PdfSegmentationServiceTest {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
|
||||
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream(), null);
|
||||
Document document = pdfSegmentationService.parseDocument(pdfFileResource.getInputStream());
|
||||
assertThat(document.getParagraphs()
|
||||
.stream()
|
||||
.flatMap(paragraph -> paragraph.getTables().stream())
|
||||
@@ -237,5 +240,12 @@ public class PdfSegmentationServiceTest {
|
||||
.equals(firstTableHeaderCells))).isTrue();
|
||||
}
|
||||
|
||||
private static String getTemporaryDirectory() {
|
||||
String tmpdir = System.getProperty("java.io.tmpdir");
|
||||
if (StringUtils.isNotBlank(tmpdir)) {
|
||||
return tmpdir;
|
||||
}
|
||||
return "/tmp";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -1660,4 +1660,3 @@ Zyma SA
|
||||
Zyma SA, Nyon, Switzerland
|
||||
Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
|
||||
Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
|
||||
Test Ignored Hint CBI_ADDRESS
|
||||
-1
@@ -85,4 +85,3 @@ Toxicology and Applied Pharmacology
|
||||
Toxicol Sci
|
||||
Toxicol Sci.
|
||||
Toxicol Sci. 1
|
||||
Test Ignored Hint Published Information
|
||||
|
||||
@@ -15,21 +15,6 @@ global Section section
|
||||
// section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
|
||||
// end
|
||||
|
||||
|
||||
rule "0: Add CBI_author from ai"
|
||||
when
|
||||
Section(aiMatchesType("CBI_author"))
|
||||
then
|
||||
section.addAiEntities("CBI_author", "recommendation_CBI_author");
|
||||
end
|
||||
|
||||
rule "0: Combine ai types CBI_author from ai"
|
||||
when
|
||||
Section(aiMatchesType("ORG"))
|
||||
then
|
||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "recommendation_CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Expand CBI Authors with firstname initials"
|
||||
when
|
||||
Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
|
||||
|
||||
-686
@@ -1,686 +0,0 @@
|
||||
{
|
||||
"dossierId": "f889853e-4bf8-49a9-aae5-c38605c6ef40",
|
||||
"fileId": "22ef63e29bb2a27db8497272336f6b32",
|
||||
"imageMetadata": [
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"logo": 1.0,
|
||||
"signature": 0.0,
|
||||
"other": 0.0,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "logo"
|
||||
},
|
||||
"position": {
|
||||
"x1": 89.88,
|
||||
"x2": 274.20000000000005,
|
||||
"y1": 716.24,
|
||||
"y2": 770.0,
|
||||
"pageNumber": 1
|
||||
},
|
||||
"geometry": {
|
||||
"width": 184.32000000000005,
|
||||
"height": 53.75999999999999
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.14298074612038092,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 3.42857142857143,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 0.999968,
|
||||
"logo": 1.6e-05,
|
||||
"other": 1.6e-05,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": -0.10000600000000001,
|
||||
"x2": 595.099994,
|
||||
"y1": -0.07998660000000002,
|
||||
"y2": 842.0800134,
|
||||
"pageNumber": 3
|
||||
},
|
||||
"geometry": {
|
||||
"width": 595.2,
|
||||
"height": 842.16
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 1.0000782051152328,
|
||||
"tooLarge": true,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.706754060986036,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 0.999872,
|
||||
"other": 7.9e-05,
|
||||
"logo": 4.8e-05,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": -0.10000600000000001,
|
||||
"x2": 595.099994,
|
||||
"y1": -0.07998660000000002,
|
||||
"y2": 842.0800134,
|
||||
"pageNumber": 7
|
||||
},
|
||||
"geometry": {
|
||||
"width": 595.2,
|
||||
"height": 842.16
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 1.0000782051152328,
|
||||
"tooLarge": true,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.706754060986036,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 0.996366,
|
||||
"other": 0.00,
|
||||
"logo": 2.3e-05,
|
||||
"formula": 4e-06
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": -0.10000600000000001,
|
||||
"x2": 595.099994,
|
||||
"y1": -0.07998660000000002,
|
||||
"y2": 842.0800134,
|
||||
"pageNumber": 8
|
||||
},
|
||||
"geometry": {
|
||||
"width": 595.2,
|
||||
"height": 842.16
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 1.0002630764355351,
|
||||
"tooLarge": true,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.706754060986036,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 0.999772,
|
||||
"logo": 0.000131,
|
||||
"other": 9.7e-05,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": 82.59443842482001,
|
||||
"x2": 512.6365568843402,
|
||||
"y1": 116.943736387567,
|
||||
"y2": 725.0718450317352,
|
||||
"pageNumber": 73
|
||||
},
|
||||
"geometry": {
|
||||
"width": 430.04211845952017,
|
||||
"height": 608.1281086441682
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.72236755521117,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.7071571143427432,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 1.0,
|
||||
"logo": 0.0,
|
||||
"other": 0.0,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": 328.20483600000006,
|
||||
"x2": 393.94460940000005,
|
||||
"y1": 175.1643178,
|
||||
"y2": 203.92865619999998,
|
||||
"pageNumber": 81
|
||||
},
|
||||
"geometry": {
|
||||
"width": 65.73977339999999,
|
||||
"height": 28.764338399999986
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.06142518774572455,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 2.2854609929078022,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 1.0,
|
||||
"logo": 0.0,
|
||||
"other": 0.0,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": 136.5955818,
|
||||
"x2": 224.72461859999999,
|
||||
"y1": 175.1133172,
|
||||
"y2": 203.97965680000001,
|
||||
"pageNumber": 81
|
||||
},
|
||||
"geometry": {
|
||||
"width": 88.1290368,
|
||||
"height": 28.866339600000003
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.07124601312700823,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 3.053003533568904,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"logo": 1.0,
|
||||
"signature": 0.0,
|
||||
"other": 0.0,
|
||||
"formula": 0.0
|
||||
},
|
||||
"label": "logo"
|
||||
},
|
||||
"position": {
|
||||
"x1": 194.99126880000003,
|
||||
"x2": 399.80967840000005,
|
||||
"y1": 554.6597824,
|
||||
"y2": 686.2413304,
|
||||
"pageNumber": 81
|
||||
},
|
||||
"geometry": {
|
||||
"width": 204.81840960000002,
|
||||
"height": 131.581548
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.23189275858788796,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 1.5565891472868219,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 133.9945512,
|
||||
"x2": 242.52382799999998,
|
||||
"y1": 411.24609519999996,
|
||||
"y2": 523.2434128,
|
||||
"pageNumber": 90
|
||||
},
|
||||
"geometry": {
|
||||
"width": 108.52927679999999,
|
||||
"height": 111.99731760000003
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.15573364968831904,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.9690346083788703,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 133.5865464,
|
||||
"x2": 242.3198256,
|
||||
"y1": 274.972492,
|
||||
"y2": 387.7858192,
|
||||
"pageNumber": 90
|
||||
},
|
||||
"geometry": {
|
||||
"width": 108.7332792,
|
||||
"height": 112.8133272
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.15644678522591335,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.9638336347197106,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 246.19587120000003,
|
||||
"x2": 356.5611696,
|
||||
"y1": 400.84197279999995,
|
||||
"y2": 519.3673672,
|
||||
"pageNumber": 90
|
||||
},
|
||||
"geometry": {
|
||||
"width": 110.3652984,
|
||||
"height": 118.52539440000004
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.1615575178049721,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.9311531841652321,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 244.9718568,
|
||||
"x2": 358.3971912,
|
||||
"y1": 274.972492,
|
||||
"y2": 387.7858192,
|
||||
"pageNumber": 90
|
||||
},
|
||||
"geometry": {
|
||||
"width": 113.4253344,
|
||||
"height": 112.8133272
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.15978662903260646,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 1.0054249547920433,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 254.9679744,
|
||||
"x2": 371.6573472,
|
||||
"y1": 439.6024288,
|
||||
"y2": 564.0438928,
|
||||
"pageNumber": 91
|
||||
},
|
||||
"geometry": {
|
||||
"width": 116.6893728,
|
||||
"height": 124.441464
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.17021718544102565,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.9377049180327869,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 133.9945512,
|
||||
"x2": 249.663912,
|
||||
"y1": 443.07046959999997,
|
||||
"y2": 687.2613424,
|
||||
"pageNumber": 91
|
||||
},
|
||||
"geometry": {
|
||||
"width": 115.66936080000002,
|
||||
"height": 244.19087280000002
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.23739910530627284,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.4736842105263158,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"other": 1.0,
|
||||
"signature": 0.0,
|
||||
"formula": 0.0,
|
||||
"logo": 0.0
|
||||
},
|
||||
"label": "other"
|
||||
},
|
||||
"position": {
|
||||
"x1": 105.84222,
|
||||
"x2": 374.870385,
|
||||
"y1": 526.40545,
|
||||
"y2": 687.05734,
|
||||
"pageNumber": 92
|
||||
},
|
||||
"geometry": {
|
||||
"width": 269.028165,
|
||||
"height": 160.65188999999998
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.2936614851112628,
|
||||
"tooLarge": false,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 1.6746031746031749,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"logo": 0.788068,
|
||||
"other": 0.152259,
|
||||
"formula": 0.036883,
|
||||
"signature": 0.02279
|
||||
},
|
||||
"label": "logo"
|
||||
},
|
||||
"position": {
|
||||
"x1": 44.64999049990001,
|
||||
"x2": 550.5759424999001,
|
||||
"y1": 63.286004150029996,
|
||||
"y2": 778.72242095003,
|
||||
"pageNumber": 94
|
||||
},
|
||||
"geometry": {
|
||||
"width": 505.92595200000005,
|
||||
"height": 715.4364168000001
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.8498341845521462,
|
||||
"tooLarge": true,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.7071571143427431,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"classification": {
|
||||
"probabilities": {
|
||||
"signature": 0.998335,
|
||||
"logo": 0.000955,
|
||||
"other": 0.000703,
|
||||
"formula": 7e-06
|
||||
},
|
||||
"label": "signature"
|
||||
},
|
||||
"position": {
|
||||
"x1": 58.954005540029996,
|
||||
"x2": 536.45979618003,
|
||||
"y1": 83.94401504006001,
|
||||
"y2": 758.05854296006,
|
||||
"pageNumber": 95
|
||||
},
|
||||
"geometry": {
|
||||
"width": 477.50579064000004,
|
||||
"height": 674.11452792
|
||||
},
|
||||
"filters": {
|
||||
"geometry": {
|
||||
"imageSize": {
|
||||
"quotient": 0.8014221863697041,
|
||||
"tooLarge": true,
|
||||
"tooSmall": false
|
||||
},
|
||||
"imageFormat": {
|
||||
"quotient": 0.7083452007974936,
|
||||
"tooTall": false,
|
||||
"tooWide": false
|
||||
}
|
||||
},
|
||||
"probability": {
|
||||
"unconfident": false
|
||||
},
|
||||
"allPassed": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Reference in New Issue
Block a user