Marmelator/persistence-service
1
0
Fork 0
Code Issues Pull Requests 17 Actions Packages Projects Releases 1.1k Wiki Activity

RED-3800 adjusted pdftron api

Browse Source
This commit is contained in:
Timo Bejan 2022-05-03 13:02:31 +03:00
parent eb438dc234
commit 4ff60781f5
18 changed files with 156 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileRequest.java → persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java
View File

@ -6,13 +6,13 @@ import lombok.*;
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class BinaryFileRequest {
public class AddFileRequest {
@NonNull
private byte[] data;
@NonNull
private String filename;
@NonNull
private String fileId;
@NonNull
private String dossierId;
private String uploader;

16
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileResult.java
View File

@ -1,16 +0,0 @@
package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file;
import lombok.*;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class BinaryFileResult {
@NonNull
private byte[] data;
@NonNull
private String filename;
}

11
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java
View File

@ -3,7 +3,16 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d
import lombok.Getter;
public enum FileType {
UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json");
UNTOUCHED(".pdf"),
ORIGIN(".pdf"),
REDACTION_LOG(".json"),
SECTION_GRID(".json"),
TEXT(".json"),
NER_ENTITIES(".json"),
IMAGE_INFO(".json"),
IMPORTED_REDACTIONS(".json"),
TEXT_HIGHLIGHTS(".json");
@Getter
private final String extension;

8
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java
View File

@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v1.api.resources;
import java.util.Set;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PathVariable;
@ -11,9 +13,6 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseStatus;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest;
@ResponseStatus(value = HttpStatus.NO_CONTENT)
@ -21,7 +20,6 @@ public interface ReanalysisResource {
String REANALYZE_PATH = "/reanalyze";
String IMPORT_REDACTIONS_PATH = "/import-redactions";
String TEXT_HIGHLIGHT_CONVERSION_PATH = "/texthighlights-conversion";
String CONVERT_TEXT_HIGHLIGHTS_PATH = "/convert-texthighlights";
String OCR_REANALYZE_PATH = "/ocr/reanalyze";
String REINDEX_PATH = "/reindex";
@ -61,7 +59,7 @@ public interface ReanalysisResource {
@PostMapping(value = IMPORT_REDACTIONS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE)
void importRedactions(@RequestBody DocumentRequest documentRequest);
void importRedactions(@RequestBody ByteContentDocument documentRequest);
@PostMapping(value = IMPORT_REDACTIONS_PATH + "/delete", consumes = MediaType.APPLICATION_JSON_VALUE)
void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest);

30
persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java
View File

@ -1,8 +1,7 @@
package com.iqser.red.service.persistence.service.v1.api.resources;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
@ -16,11 +15,6 @@ public interface UploadResource {
String DELETE_PATH = REST_PATH + "/delete";
String UPLOAD_PATH = REST_PATH + "/upload";
String ORIGINAL_PATH = REST_PATH + "/original";
String ANNOTATED_PATH = REST_PATH + "/annotated";
String REDACTED_PATH = REST_PATH + "/redacted";
String PREVIEW_PATH = REST_PATH + "/preview";
String FLATTED_PATH = REST_PATH + "/flatted";
String HARD_DELETE_PATH = REST_PATH + "/hardDelete";
String UNDELETE_PATH = REST_PATH + "/undelete";
@ -30,10 +24,12 @@ public interface UploadResource {
String DOSSIER_ID_PARAM = "dossierId";
String DOSSIER_ID_PATH_PARAM = "/{" + DOSSIER_ID_PARAM + "}";
String FLATTEN_PARAM = "flatten";
@PostMapping(value = UPLOAD_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces =
MediaType.APPLICATION_JSON_VALUE)
JSONPrimitive<String> upload(@RequestBody BinaryFileRequest request);
JSONPrimitive<String> upload(@RequestBody AddFileRequest request);
@DeleteMapping(value = DELETE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE)
@ -48,22 +44,4 @@ public interface UploadResource {
void undeleteFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set<String> fileIds);
@PostMapping(value = ORIGINAL_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
BinaryFileResult getOriginal(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@PostMapping(value = ANNOTATED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
BinaryFileResult getAnnotated(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@PostMapping(value = REDACTED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
BinaryFileResult getRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@PostMapping(value = PREVIEW_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
BinaryFileResult getPreview(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
@PostMapping(value = FLATTED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE)
BinaryFileResult getFlatRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId);
}

52
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java
View File

@ -1,28 +1,10 @@
package com.iqser.red.service.peristence.v1.server.controller;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.*;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation;
import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.peristence.v1.server.service.IndexingService;
@ -36,10 +18,21 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.time.OffsetDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
@RestController
@ -113,21 +106,12 @@ public class ReanalysisController implements ReanalysisResource {
}
public void importRedactions(@RequestBody DocumentRequest documentRequest) {
// validate the pdf file
PdfTronOptimizeRequest request = new PdfTronOptimizeRequest();
request.setDocument(documentRequest.getDocument());
PdfTronOptimizeResponse optimize;
public void importRedactions(@RequestBody ByteContentDocument documentRequest) {
try {
optimize = pDFTronRedactionClient.optimize(request);
pDFTronRedactionClient.importRedactions(documentRequest);
} catch (Exception e) {
throw new BadRequestException("File Not Valid PDF");
throw new BadRequestException("File Not Valid PDF or number of pages do not match");
}
FileModel file = fileStatusService.getStatus(documentRequest.getFileId());
if (optimize != null && optimize.getNumberOfPages() != file.getNumberOfPages()) {
throw new BadRequestException("The files differ in number of pages");
}
pDFTronRedactionClient.importRedactions(documentRequest);
fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false);
}

99
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java
View File

@ -1,28 +1,14 @@
package com.iqser.red.service.peristence.v1.server.controller;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionResult;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.service.DossierService;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.FileService;
import com.iqser.red.service.peristence.v1.server.service.FileStatusService;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
@ -41,12 +27,9 @@ public class UploadController implements UploadResource {
private final FileService fileService;
private final FileStatusService fileStatusService;
private final DossierService dossierService;
private final PDFTronRedactionClient pdfTronRedactionClient;
private final FileManagementStorageService fileManagementStorageService;
private final RedactionClient redactionClient;
@Override
public JSONPrimitive<String> upload(@RequestBody BinaryFileRequest request) {
public JSONPrimitive<String> upload(@RequestBody AddFileRequest request) {
return fileService.upload(request);
}
@ -98,82 +81,4 @@ public class UploadController implements UploadResource {
}
}
@Override
public BinaryFileResult getOriginal(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
var fileStatus = fileStatusService.getStatus(fileId);
if (fileStatus.getDeleted() != null) {
throw new NotFoundException("File is deleted.");
}
try {
return new BinaryFileResult(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN), fileStatus.getFilename());
} catch (StorageObjectDoesNotExist e) {
throw new NotFoundException(e.getMessage());
}
}
@Override
public BinaryFileResult getAnnotated(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
var fileStatus = fileStatusService.getStatus(fileId);
var dossier = dossierService.getDossierById(dossierId);
AnnotateResponse annotateResponse = redactionClient.annotate(AnnotateRequest.builder()
.fileId(fileId)
.dossierId(dossierId)
.dossierTemplateId(dossier.getDossierTemplateId())
.build());
return new BinaryFileResult(annotateResponse.getDocument(), fileStatus.getFilename());
}
@Override
@SneakyThrows
public BinaryFileResult getRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
var fileStatus = fileStatusService.getStatus(fileId);
var dossier = dossierService.getDossierById(dossierId);
var request = PdfTronRedactionRequest.builder()
.dossierTemplateId(dossier.getDossierTemplateId())
.applyWatermark(dossier.isWatermarkEnabled())
.document(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN))
.redactionLog(fileManagementStorageService.getRedactionLog(dossierId, fileId))
.build();
PdfTronRedactionResult pdfTronRedactionResult = pdfTronRedactionClient.redact(request);
return new BinaryFileResult(pdfTronRedactionResult.getDocument(), fileStatus.getFilename());
}
@Override
@SneakyThrows
public BinaryFileResult getPreview(String dossierId, String fileId) {
var fileStatus = fileStatusService.getStatus(fileId);
var dossier = dossierService.getDossierById(dossierId);
PdfTronRedactionResult pdfTronRedactionResult = pdfTronRedactionClient.redactionPreview(PdfTronRedactionRequest.builder()
.dossierTemplateId(dossier.getDossierTemplateId())
.applyWatermark(dossier.isWatermarkPreviewEnabled())
.document(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN))
.redactionLog(fileManagementStorageService.getRedactionLog(dossierId, fileId))
.build());
return new BinaryFileResult(pdfTronRedactionResult.getDocument(), fileStatus.getFilename());
}
@Override
public BinaryFileResult getFlatRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) {
var redacted = getRedacted(dossierId, fileId);
PdfTronFlattenResponse flattenResponse = pdfTronRedactionClient.flatDocument(PdfTronFlattenRequest.builder()
.document(redacted.getData())
.build());
return BinaryFileResult.builder().filename(redacted.getFilename()).data(flattenResponse.getDocument()).build();
}
}

23
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java
View File

@ -1,6 +1,6 @@
package com.iqser.red.service.peristence.v1.server.migration.migrations;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.peristence.v1.server.migration.Migration;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
@ -48,9 +48,24 @@ public class MigrateHighlights3 extends Migration {
files.forEach(file -> {
if (file.getHardDeletedTime() == null) {
var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN);
var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes));
fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights);
var untouchedExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.UNTOUCHED);
if (!untouchedExists) {
var originExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.ORIGIN);
if (!originExists) {
log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId());
return;
}
fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED,
fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN));
}
var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder()
.fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build());
fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights());
}
});
});

18
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java
View File

@ -101,25 +101,11 @@ public class FileManagementStorageService {
}
}
public boolean typeExists(String dossierId, String fileId, FileType type) {
public boolean imageInfoExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO));
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, type));
}
public boolean textExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT));
}
public boolean nerEntitiesExists(String dossierId, String fileId) {
return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES));
}
public void deleteObject(String dossierId, String fileId, FileType fileType) {
storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType));

27
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java
View File

@ -2,7 +2,8 @@ package com.iqser.red.service.peristence.v1.server.service;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient;
import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException;
import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException;
@ -11,7 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
@ -33,8 +34,6 @@ import java.util.Arrays;
@RequiredArgsConstructor
public class FileService {
private final HashFunction hashFunction = Hashing.murmur3_128();
private final AddRedactionPersistenceService addRedactionPersistenceService;
private final CommentPersistenceService commentPersistenceService;
private final ForceRedactionPersistenceService forceRedactionPersistenceService;
@ -50,12 +49,11 @@ public class FileService {
private final IndexingService indexingService;
public JSONPrimitive<String> upload(BinaryFileRequest request) {
public JSONPrimitive<String> upload(AddFileRequest request) {
dossierPersistenceService.getAndValidateDossier(request.getDossierId());
var fileId = generateFileId(request);
var existingStatus = retrieveStatus(fileId);
var existingStatus = retrieveStatus(request.getFileId());
if (existingStatus != null && !existingStatus.isSoftOrHardDeleted()) {
if (existingStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) {
@ -68,9 +66,10 @@ public class FileService {
}
}
boolean hasHighlights;
UntouchedDocumentResponse untouchedDocumentResponse;
try {
hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getFilename(), request.getData()));
untouchedDocumentResponse = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder().dossierId(request.getDossierId())
.fileId(request.getFileId()).fileName(request.getFilename()).build());
} catch (FeignException e) {
log.warn("Failed to optimize file: {}", request.getFilename(), e);
@ -80,13 +79,13 @@ public class FileService {
if (existingStatus != null) {
// the file is already uploaded, just reanalyse it.
fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights());
} else {
// the file is new, should create a new status for it.
log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename());
fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights);
fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights());
}
return new JSONPrimitive<>(fileId);
return new JSONPrimitive<>(request.getFileId());
}
@ -303,8 +302,8 @@ public class FileService {
}
private String generateFileId(BinaryFileRequest file) {
private final HashFunction hashFunction = Hashing.murmur3_128();
private String generateFileId(AddFileRequest file) {
return hashFunction.hashBytes((file.getFilename() + file.getDossierId()).getBytes(StandardCharsets.UTF_8))
.toString();

3
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java
View File

@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.service;
import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.search.v1.model.IndexMessageType;
import lombok.RequiredArgsConstructor;
@ -31,7 +32,7 @@ public class FileStatusProcessingUpdateService {
switch (analyzeResult.getMessageType()) {
case STRUCTURE_ANALYSE:
if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
if (settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
fileStatusService.addToNerQueue(dossierId, fileId);
} else {

13
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java
View File

@ -10,6 +10,7 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
@ -18,7 +19,6 @@ import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage;
import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration;
import com.iqser.red.service.peristence.v1.server.controller.RulesController;
import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest;
@ -224,7 +224,7 @@ public class FileStatusService {
boolean reanalyse = isReanalyse(dossier, fileStatus);
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) {
if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId,FileType.IMAGE_INFO)) {
log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId);
addToImageQueue(dossierId, fileId);
return;
@ -232,7 +232,7 @@ public class FileStatusService {
MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus());
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) {
if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) {
log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId);
addToNerQueue(dossierId, fileId);
return;
@ -276,7 +276,8 @@ public class FileStatusService {
return !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus()
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
.equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId())
&& (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed()
.isAfter(fileStatus.getLastFileAttributeChange()));
}
@ -284,7 +285,7 @@ public class FileStatusService {
private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse,
ProcessingStatus processingStatus) {
if (!fileManagementStorageService.textExists(dossierId, fileId)) {
if (!fileManagementStorageService.typeExists(dossierId, fileId, FileType.TEXT)) {
return MessageType.STRUCTURE_ANALYSE;
}
if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) {
@ -437,7 +438,7 @@ public class FileStatusService {
private void addToOcrQueue(String dossierId, String fileId, int priority) {
try {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new OcrRequestMessage(dossierId, fileId)), message -> {
rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> {
message.getMessageProperties().setPriority(priority);
return message;
});

52
persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java
View File

@ -1,13 +1,6 @@
package com.iqser.red.service.peristence.v1.server.service.download;
import java.util.Map;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.stereotype.Service;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.peristence.v1.server.service.RedactionLogService;
@ -18,11 +11,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.
import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity;
import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity;
import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*;
import com.iqser.red.service.persistence.service.v1.api.model.audit.AddNotificationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
@ -31,9 +20,13 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage;
import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation;
import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import javax.transaction.Transactional;
import java.util.Map;
import java.util.stream.Collectors;
@Slf4j
@Service
@ -94,21 +87,17 @@ public class DownloadPreparationService {
if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original));
}
if (downloadFileType.name().equals(DownloadFileType.ANNOTATED.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Annotated", addSuffix(fileStatus.getFilename(), "annotated"),
getAnnotated(dossier.getDossierTemplateId(), fileStatus.getDossierId(), fileId)));
}
if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"),
getPreview(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled())));
getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled())));
}
if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"),
getDeltaPreview(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled())));
getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled())));
}
if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) {
fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"),
getRedacted(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkEnabled())));
getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkEnabled())));
}
}
log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds
@ -195,11 +184,12 @@ public class DownloadPreparationService {
}
private byte[] getRedacted(byte[] original, RedactionLog redactionLog, String dossierTemplateId,
private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId,
boolean watermarkEnabled) {
return pdfTronRedactionClient.redact(PdfTronRedactionRequest.builder()
.document(original)
return pdfTronRedactionClient.redact(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.applyWatermark(watermarkEnabled)
@ -207,10 +197,11 @@ public class DownloadPreparationService {
}
private byte[] getPreview(byte[] original, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) {
private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) {
return pdfTronRedactionClient.redactionPreview(PdfTronRedactionRequest.builder()
.document(original)
return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.applyWatermark(watermarkEnabled)
@ -218,10 +209,11 @@ public class DownloadPreparationService {
}
private byte[] getDeltaPreview(byte[] original, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) {
private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) {
return pdfTronRedactionClient.redactionPreviewDiff(PdfTronRedactionRequest.builder()
.document(original)
return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder()
.dossierId(dossierId)
.fileId(fileId)
.dossierTemplateId(dossierTemplateId)
.redactionLog(redactionLog)
.applyWatermark(watermarkEnabled)

7
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java
View File

@ -7,13 +7,14 @@ import com.iqser.red.service.peristence.v1.server.integration.client.UploadClien
import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionGrid;
import lombok.SneakyThrows;
import org.apache.commons.codec.binary.Base64;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -66,8 +67,10 @@ public class FileTesterAndProvider {
@SneakyThrows
public String testAndProvideFileQuick(Dossier dossier, String fileName) {
var fileId = Base64.encodeBase64String((dossier.getId() + fileName).getBytes(StandardCharsets.UTF_8));
BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1");
AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1");
fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8));
JSONPrimitive<String> uploadResult = uploadClient.upload(upload);
return uploadResult.getValue();

56
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java
View File

@ -1,35 +1,26 @@
package com.iqser.red.service.peristence.v1.server.integration.tests;
import static org.assertj.core.api.Assertions.assertThat;
import java.nio.charset.StandardCharsets;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.List;
import com.iqser.red.service.peristence.v1.server.integration.client.*;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import org.junit.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.common.collect.Sets;
import com.iqser.red.service.peristence.v1.server.integration.client.*;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.*;
import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus;
import org.junit.Test;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import feign.FeignException;
import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class FileTest extends AbstractPersistenceServerServiceTest {
@ -60,20 +51,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
@Autowired
private ViewedPagesClient viewedPagesClient;
@Test
public void testEmptyFile() {
var dossier = dossierTesterAndProvider.provideTestDossier();
BinaryFileRequest upload = new BinaryFileRequest("".getBytes(StandardCharsets.UTF_8), "emptyFile.pdf", dossier.getId(), "1");
try {
uploadClient.upload(upload);
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
}
@Test
public void testFileSoftDeleteReupload() {
@ -122,7 +99,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(viewedPages.size()).isEqualTo(1);
BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), filename, dossier.getId(), "1");
AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1");
JSONPrimitive<String> uploadResult = uploadClient.upload(upload);
loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue());
@ -229,13 +206,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest {
assertThat(loadedFile.getLastApprover()).isEqualTo("2");
assertThat(uploadClient.getOriginal(dossier.getId(), file.getId())).isNotNull();
assertThat(uploadClient.getRedacted(dossier.getId(), file.getId())).isNotNull();
assertThat(uploadClient.getFlatRedacted(dossier.getId(), file.getId())).isNotNull();
assertThat(uploadClient.getPreview(dossier.getId(), file.getId())).isNotNull();
assertThat(uploadClient.getAnnotated(dossier.getId(), file.getId())).isNotNull();
fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null);

28
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java
View File

@ -7,20 +7,18 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes
import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider;
import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel;
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus;
import feign.FeignException;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
@ -74,30 +72,6 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest {
}
@Test
public void testImportRedactions() {
var dossier = dossierTesterAndProvider.provideTestDossier();
var file = fileTesterAndProvider.testAndProvideFile(dossier);
BinaryFileRequest upload = new BinaryFileRequest("testing".getBytes(StandardCharsets.UTF_8), "emptyFile.pdf", dossier.getId(), "1");
DocumentRequest documentRequest = DocumentRequest.builder()
.document(upload.getData())
.fileId(file.getId()).build();
when(pdfTronRedactionClient.optimize(Mockito.any())).thenAnswer((args) ->
PdfTronOptimizeResponse.builder()
.document(((PdfTronOptimizeRequest) args.getArguments()[0]).getDocument())
.numberOfPages(100).build());
try {
reanalysisClient.importRedactions(documentRequest);
} catch (FeignException e) {
assertThat(e.status()).isEqualTo(400);
}
}
private void resetProcessingStatus(FileModel file) {

44
persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java
View File

@ -5,6 +5,8 @@ import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.when;
import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument;
import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse;
import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*;
import org.assertj.core.util.Lists;
import org.junit.After;
@ -31,10 +33,6 @@ import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenResponse;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest;
import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionResult;
import com.iqser.red.service.peristence.v1.server.Application;
import com.iqser.red.service.peristence.v1.server.client.RedactionClient;
import com.iqser.red.service.peristence.v1.server.client.SearchClient;
@ -48,6 +46,7 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionResult;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
@RunWith(SpringRunner.class)
@EnableFeignClients(basePackageClasses = FileClient.class)
@ -149,19 +148,38 @@ public abstract class AbstractPersistenceServerServiceTest {
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), d.getDocument());
return null;
}).when(pdfTronRedactionClient).saveOptimizedAndAnnotations(any());
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.UNTOUCHED);
var untouched = storageService.getObject(untouchedObjectId);
storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), IOUtils.toByteArray(untouched.getInputStream()));
return UntouchedDocumentResponse.builder().build();
}).when(pdfTronRedactionClient).processUntouchedDocument(any());
when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((args) ->
new PdfTronRedactionResult(((PdfTronRedactionRequest) args.getArguments()[0]).getDocument()));
when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((args) ->
new PdfTronRedactionResult(((PdfTronRedactionRequest) args.getArguments()[0]).getDocument()));
when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) ->{
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream()));
});
when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((answer) ->{
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream()));
});
when(pdfTronRedactionClient.redactionPreviewDiff(Mockito.any())).thenAnswer((answer) ->{
Object[] args = answer.getArguments();
DocumentRequest d = (DocumentRequest) args[0];
var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN);
return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream()));
});
when(pdfTronRedactionClient.flatDocument(Mockito.any())).thenAnswer((args) ->
new PdfTronFlattenResponse(((PdfTronFlattenRequest) args.getArguments()[0]).getDocument()));
when(redactionClient.annotate(Mockito.any())).thenReturn(new AnnotateResponse("document".getBytes()));
when(redactionClient.sections(Mockito.any())).thenReturn(new RedactionResult());

2
persistence-service-v1/pom.xml
View File

@ -27,7 +27,7 @@
<properties>
<redaction-service.version>3.108.0</redaction-service.version>
<search-service.version>2.36.0</search-service.version>
<pdftron-redaction-service.version>3.75.0</pdftron-redaction-service.version>
<pdftron-redaction-service.version>3.79.0</pdftron-redaction-service.version>
<redaction-report-service.version>3.45.0</redaction-report-service.version>
</properties>