From 4ff60781f54017ef50682f69f00515ed89b27da2 Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Tue, 3 May 2022 13:02:31 +0300 Subject: [PATCH] RED-3800 adjusted pdftron api --- ...ryFileRequest.java => AddFileRequest.java} | 6 +- .../dossier/file/BinaryFileResult.java | 16 --- .../dossier/file/FileType.java | 11 ++- .../v1/api/resources/ReanalysisResource.java | 8 +- .../v1/api/resources/UploadResource.java | 30 +----- .../controller/ReanalysisController.java | 52 ++++------ .../server/controller/UploadController.java | 99 +------------------ .../migrations/MigrateHighlights3.java | 23 ++++- .../service/FileManagementStorageService.java | 18 +--- .../v1/server/service/FileService.java | 27 +++-- .../FileStatusProcessingUpdateService.java | 3 +- .../v1/server/service/FileStatusService.java | 13 +-- .../download/DownloadPreparationService.java | 52 +++++----- .../service/FileTesterAndProvider.java | 7 +- .../v1/server/integration/tests/FileTest.java | 56 +++-------- .../integration/tests/ReanalysisTest.java | 28 +----- .../AbstractPersistenceServerServiceTest.java | 44 ++++++--- persistence-service-v1/pom.xml | 2 +- 18 files changed, 156 insertions(+), 339 deletions(-) rename persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/{BinaryFileRequest.java => AddFileRequest.java} (84%) delete mode 100644 persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileResult.java diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileRequest.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java similarity index 84% rename from persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileRequest.java rename to persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java index 624a2997c..4c00841b6 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileRequest.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/AddFileRequest.java @@ -6,13 +6,13 @@ import lombok.*; @Builder @AllArgsConstructor @NoArgsConstructor(access = AccessLevel.PRIVATE) -public class BinaryFileRequest { +public class AddFileRequest { - @NonNull - private byte[] data; @NonNull private String filename; @NonNull + private String fileId; + @NonNull private String dossierId; private String uploader; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileResult.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileResult.java deleted file mode 100644 index 02929b66e..000000000 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/BinaryFileResult.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file; - -import lombok.*; - -@Data -@Builder -@AllArgsConstructor -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class BinaryFileResult { - - @NonNull - private byte[] data; - - @NonNull - private String filename; -} diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java index def75e2ca..0e1579b21 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/model/dossiertemplate/dossier/file/FileType.java @@ -3,7 +3,16 @@ package com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.d import lombok.Getter; public enum FileType { - UNTOUCHED(".pdf"), ORIGIN(".pdf"), REDACTION_LOG(".json"), SECTION_GRID(".json"), TEXT(".json"), NER_ENTITIES(".json"), IMAGE_INFO(".json"), IMPORTED_REDACTIONS(".json"), TEXT_HIGHLIGHTS(".json"); + + UNTOUCHED(".pdf"), + ORIGIN(".pdf"), + REDACTION_LOG(".json"), + SECTION_GRID(".json"), + TEXT(".json"), + NER_ENTITIES(".json"), + IMAGE_INFO(".json"), + IMPORTED_REDACTIONS(".json"), + TEXT_HIGHLIGHTS(".json"); @Getter private final String extension; diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java index cc916ec97..b2f9ab0dc 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/ReanalysisResource.java @@ -2,6 +2,8 @@ package com.iqser.red.service.persistence.service.v1.api.resources; import java.util.Set; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.PathVariable; @@ -11,9 +13,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseStatus; import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse; import com.iqser.red.service.persistence.service.v1.api.model.annotations.DeleteImportedRedactionsRequest; @ResponseStatus(value = HttpStatus.NO_CONTENT) @@ -21,7 +20,6 @@ public interface ReanalysisResource { String REANALYZE_PATH = "/reanalyze"; String IMPORT_REDACTIONS_PATH = "/import-redactions"; - String TEXT_HIGHLIGHT_CONVERSION_PATH = "/texthighlights-conversion"; String CONVERT_TEXT_HIGHLIGHTS_PATH = "/convert-texthighlights"; String OCR_REANALYZE_PATH = "/ocr/reanalyze"; String REINDEX_PATH = "/reindex"; @@ -61,7 +59,7 @@ public interface ReanalysisResource { @PostMapping(value = IMPORT_REDACTIONS_PATH, consumes = MediaType.APPLICATION_JSON_VALUE) - void importRedactions(@RequestBody DocumentRequest documentRequest); + void importRedactions(@RequestBody ByteContentDocument documentRequest); @PostMapping(value = IMPORT_REDACTIONS_PATH + "/delete", consumes = MediaType.APPLICATION_JSON_VALUE) void deleteImportedRedactions(@RequestBody DeleteImportedRedactionsRequest deleteImportedRedactionsRequest); diff --git a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java index 9653a0064..0982b26af 100644 --- a/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java +++ b/persistence-service-v1/persistence-service-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/resources/UploadResource.java @@ -1,8 +1,7 @@ package com.iqser.red.service.persistence.service.v1.api.resources; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.*; @@ -16,11 +15,6 @@ public interface UploadResource { String DELETE_PATH = REST_PATH + "/delete"; String UPLOAD_PATH = REST_PATH + "/upload"; - String ORIGINAL_PATH = REST_PATH + "/original"; - String ANNOTATED_PATH = REST_PATH + "/annotated"; - String REDACTED_PATH = REST_PATH + "/redacted"; - String PREVIEW_PATH = REST_PATH + "/preview"; - String FLATTED_PATH = REST_PATH + "/flatted"; String HARD_DELETE_PATH = REST_PATH + "/hardDelete"; String UNDELETE_PATH = REST_PATH + "/undelete"; @@ -30,10 +24,12 @@ public interface UploadResource { String DOSSIER_ID_PARAM = "dossierId"; String DOSSIER_ID_PATH_PARAM = "/{" + DOSSIER_ID_PARAM + "}"; + String FLATTEN_PARAM = "flatten"; + @PostMapping(value = UPLOAD_PATH, consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE) - JSONPrimitive upload(@RequestBody BinaryFileRequest request); + JSONPrimitive upload(@RequestBody AddFileRequest request); @DeleteMapping(value = DELETE_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE) @@ -48,22 +44,4 @@ public interface UploadResource { void undeleteFiles(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @RequestBody Set fileIds); - @PostMapping(value = ORIGINAL_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) - BinaryFileResult getOriginal(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); - - - @PostMapping(value = ANNOTATED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) - BinaryFileResult getAnnotated(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); - - - @PostMapping(value = REDACTED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) - BinaryFileResult getRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); - - @PostMapping(value = PREVIEW_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) - BinaryFileResult getPreview(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); - - - @PostMapping(value = FLATTED_PATH + DOSSIER_ID_PATH_PARAM + FILE_ID_PATH_VARIABLE, produces = MediaType.APPLICATION_JSON_VALUE) - BinaryFileResult getFlatRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId); - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java index fc62883af..cbab36a13 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/ReanalysisController.java @@ -1,28 +1,10 @@ package com.iqser.red.service.peristence.v1.server.controller; -import java.time.OffsetDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronOptimizeResponse; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionOperation; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightConversionRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightOperation; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.TextHighlightResponse; +import com.iqser.red.service.pdftron.redaction.v1.api.model.*; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionOperation; +import com.iqser.red.service.pdftron.redaction.v1.api.model.highlights.TextHighlightConversionRequest; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.service.FileStatusService; import com.iqser.red.service.peristence.v1.server.service.IndexingService; @@ -36,10 +18,21 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; import com.iqser.red.service.persistence.service.v1.api.resources.ReanalysisResource; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.time.OffsetDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; @Slf4j @RestController @@ -113,21 +106,12 @@ public class ReanalysisController implements ReanalysisResource { } - public void importRedactions(@RequestBody DocumentRequest documentRequest) { - // validate the pdf file - PdfTronOptimizeRequest request = new PdfTronOptimizeRequest(); - request.setDocument(documentRequest.getDocument()); - PdfTronOptimizeResponse optimize; + public void importRedactions(@RequestBody ByteContentDocument documentRequest) { try { - optimize = pDFTronRedactionClient.optimize(request); + pDFTronRedactionClient.importRedactions(documentRequest); } catch (Exception e) { - throw new BadRequestException("File Not Valid PDF"); + throw new BadRequestException("File Not Valid PDF or number of pages do not match"); } - FileModel file = fileStatusService.getStatus(documentRequest.getFileId()); - if (optimize != null && optimize.getNumberOfPages() != file.getNumberOfPages()) { - throw new BadRequestException("The files differ in number of pages"); - } - pDFTronRedactionClient.importRedactions(documentRequest); fileStatusService.setStatusFullReprocess(documentRequest.getDossierId(), documentRequest.getFileId(), true, false); } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java index c36ce3637..50f962848 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/controller/UploadController.java @@ -1,28 +1,14 @@ package com.iqser.red.service.peristence.v1.server.controller; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenResponse; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionResult; -import com.iqser.red.service.peristence.v1.server.client.RedactionClient; import com.iqser.red.service.peristence.v1.server.service.DossierService; -import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.service.FileService; import com.iqser.red.service.peristence.v1.server.service.FileStatusService; -import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.exception.DossierNotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileResult; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.resources.UploadResource; -import com.iqser.red.service.redaction.v1.model.AnnotateRequest; -import com.iqser.red.service.redaction.v1.model.AnnotateResponse; -import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @@ -41,12 +27,9 @@ public class UploadController implements UploadResource { private final FileService fileService; private final FileStatusService fileStatusService; private final DossierService dossierService; - private final PDFTronRedactionClient pdfTronRedactionClient; - private final FileManagementStorageService fileManagementStorageService; - private final RedactionClient redactionClient; @Override - public JSONPrimitive upload(@RequestBody BinaryFileRequest request) { + public JSONPrimitive upload(@RequestBody AddFileRequest request) { return fileService.upload(request); } @@ -98,82 +81,4 @@ public class UploadController implements UploadResource { } } - - @Override - public BinaryFileResult getOriginal(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - var fileStatus = fileStatusService.getStatus(fileId); - - if (fileStatus.getDeleted() != null) { - throw new NotFoundException("File is deleted."); - } - - try { - return new BinaryFileResult(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN), fileStatus.getFilename()); - } catch (StorageObjectDoesNotExist e) { - throw new NotFoundException(e.getMessage()); - } - } - - @Override - public BinaryFileResult getAnnotated(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - - var fileStatus = fileStatusService.getStatus(fileId); - var dossier = dossierService.getDossierById(dossierId); - AnnotateResponse annotateResponse = redactionClient.annotate(AnnotateRequest.builder() - .fileId(fileId) - .dossierId(dossierId) - .dossierTemplateId(dossier.getDossierTemplateId()) - .build()); - - return new BinaryFileResult(annotateResponse.getDocument(), fileStatus.getFilename()); - } - - @Override - @SneakyThrows - public BinaryFileResult getRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - - var fileStatus = fileStatusService.getStatus(fileId); - var dossier = dossierService.getDossierById(dossierId); - - var request = PdfTronRedactionRequest.builder() - .dossierTemplateId(dossier.getDossierTemplateId()) - .applyWatermark(dossier.isWatermarkEnabled()) - .document(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN)) - .redactionLog(fileManagementStorageService.getRedactionLog(dossierId, fileId)) - .build(); - - PdfTronRedactionResult pdfTronRedactionResult = pdfTronRedactionClient.redact(request); - - return new BinaryFileResult(pdfTronRedactionResult.getDocument(), fileStatus.getFilename()); - } - - @Override - @SneakyThrows - public BinaryFileResult getPreview(String dossierId, String fileId) { - - var fileStatus = fileStatusService.getStatus(fileId); - var dossier = dossierService.getDossierById(dossierId); - - PdfTronRedactionResult pdfTronRedactionResult = pdfTronRedactionClient.redactionPreview(PdfTronRedactionRequest.builder() - .dossierTemplateId(dossier.getDossierTemplateId()) - .applyWatermark(dossier.isWatermarkPreviewEnabled()) - .document(fileManagementStorageService.getStoredObjectBytes(dossierId, fileId, FileType.ORIGIN)) - .redactionLog(fileManagementStorageService.getRedactionLog(dossierId, fileId)) - .build()); - - return new BinaryFileResult(pdfTronRedactionResult.getDocument(), fileStatus.getFilename()); - } - - - @Override - public BinaryFileResult getFlatRedacted(@PathVariable(DOSSIER_ID_PARAM) String dossierId, @PathVariable(FILE_ID) String fileId) { - - var redacted = getRedacted(dossierId, fileId); - - PdfTronFlattenResponse flattenResponse = pdfTronRedactionClient.flatDocument(PdfTronFlattenRequest.builder() - .document(redacted.getData()) - .build()); - return BinaryFileResult.builder().filename(redacted.getFilename()).data(flattenResponse.getDocument()).build(); - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java index 61434eb7c..d0cbabf78 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/migration/migrations/MigrateHighlights3.java @@ -1,6 +1,6 @@ package com.iqser.red.service.peristence.v1.server.migration.migrations; -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; import com.iqser.red.service.peristence.v1.server.migration.Migration; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; @@ -48,9 +48,24 @@ public class MigrateHighlights3 extends Migration { files.forEach(file -> { if (file.getHardDeletedTime() == null) { - var fileBytes = fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN); - var hasHighlights = pdfTronRedactionClient.extractHighlights(new DocumentRequest(dossier.getId(), file.getId(), file.getFilename(), fileBytes)); - fileStatusPersistenceService.updateHasHighlights(file.getId(), hasHighlights); + var untouchedExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.UNTOUCHED); + + if (!untouchedExists) { + var originExists = fileManagementStorageService.typeExists(dossier.getId(), file.getId(), FileType.ORIGIN); + + if (!originExists) { + log.warn("Invalid file {} / {} Neither untouched nor origin files exists!", dossier.getId(), file.getId()); + return; + } + + fileManagementStorageService.storeObject(dossier.getId(), file.getId(), FileType.UNTOUCHED, + fileManagementStorageService.getStoredObjectBytes(dossier.getId(), file.getId(), FileType.ORIGIN)); + + } + var response = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder() + .fileName(file.getFilename()).fileId(file.getId()).dossierId(file.getDossierId()).build()); + + fileStatusPersistenceService.updateHasHighlights(file.getId(), response.isHasHighlights()); } }); }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java index 8f3f7b322..2862dc876 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileManagementStorageService.java @@ -101,25 +101,11 @@ public class FileManagementStorageService { } } + public boolean typeExists(String dossierId, String fileId, FileType type) { - public boolean imageInfoExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMAGE_INFO)); + return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, type)); } - - public boolean textExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT)); - } - - - public boolean nerEntitiesExists(String dossierId, String fileId) { - - return storageService.objectExists(StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES)); - } - - public void deleteObject(String dossierId, String fileId, FileType fileType) { storageService.deleteObject(StorageIdUtils.getStorageId(dossierId, fileId, fileType)); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java index 435119855..25719f418 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileService.java @@ -2,7 +2,8 @@ package com.iqser.red.service.peristence.v1.server.service; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ProcessUntouchedDocumentRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.persistence.management.v1.processor.client.PDFTronRedactionClient; import com.iqser.red.service.persistence.management.v1.processor.exception.BadRequestException; import com.iqser.red.service.persistence.management.v1.processor.exception.ConflictException; @@ -11,7 +12,7 @@ import com.iqser.red.service.persistence.management.v1.processor.service.persist import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; @@ -33,8 +34,6 @@ import java.util.Arrays; @RequiredArgsConstructor public class FileService { - private final HashFunction hashFunction = Hashing.murmur3_128(); - private final AddRedactionPersistenceService addRedactionPersistenceService; private final CommentPersistenceService commentPersistenceService; private final ForceRedactionPersistenceService forceRedactionPersistenceService; @@ -50,12 +49,11 @@ public class FileService { private final IndexingService indexingService; - public JSONPrimitive upload(BinaryFileRequest request) { + public JSONPrimitive upload(AddFileRequest request) { dossierPersistenceService.getAndValidateDossier(request.getDossierId()); - var fileId = generateFileId(request); - var existingStatus = retrieveStatus(fileId); + var existingStatus = retrieveStatus(request.getFileId()); if (existingStatus != null && !existingStatus.isSoftOrHardDeleted()) { if (existingStatus.getWorkflowStatus().equals(WorkflowStatus.APPROVED)) { @@ -68,9 +66,10 @@ public class FileService { } } - boolean hasHighlights; + UntouchedDocumentResponse untouchedDocumentResponse; try { - hasHighlights = pdfTronRedactionClient.saveOptimizedAndAnnotations(new DocumentRequest(request.getDossierId(), fileId, request.getFilename(), request.getData())); + untouchedDocumentResponse = pdfTronRedactionClient.processUntouchedDocument(ProcessUntouchedDocumentRequest.builder().dossierId(request.getDossierId()) + .fileId(request.getFileId()).fileName(request.getFilename()).build()); } catch (FeignException e) { log.warn("Failed to optimize file: {}", request.getFilename(), e); @@ -80,13 +79,13 @@ public class FileService { if (existingStatus != null) { // the file is already uploaded, just reanalyse it. - fileStatusService.overwriteFile(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights); + fileStatusService.overwriteFile(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights()); } else { // the file is new, should create a new status for it. log.info("File {} has no status yet, creating one and setting to unprocessed.", request.getFilename()); - fileStatusService.createStatus(request.getDossierId(), fileId, request.getUploader(), request.getFilename(), hasHighlights); + fileStatusService.createStatus(request.getDossierId(), request.getFileId(), request.getUploader(), request.getFilename(), untouchedDocumentResponse.isHasHighlights()); } - return new JSONPrimitive<>(fileId); + return new JSONPrimitive<>(request.getFileId()); } @@ -303,8 +302,8 @@ public class FileService { } - - private String generateFileId(BinaryFileRequest file) { + private final HashFunction hashFunction = Hashing.murmur3_128(); + private String generateFileId(AddFileRequest file) { return hashFunction.hashBytes((file.getFilename() + file.getDossierId()).getBytes(StandardCharsets.UTF_8)) .toString(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java index 026817b65..66993e0a0 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusProcessingUpdateService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.peristence.v1.server.service; import com.iqser.red.service.peristence.v1.server.settings.FileManagementServiceSettings; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.model.AnalyzeResult; import com.iqser.red.service.search.v1.model.IndexMessageType; import lombok.RequiredArgsConstructor; @@ -31,7 +32,7 @@ public class FileStatusProcessingUpdateService { switch (analyzeResult.getMessageType()) { case STRUCTURE_ANALYSE: - if (settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) { fileStatusService.addToNerQueue(dossierId, fileId); } else { diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java index 03b28fa55..c9699d8d4 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/FileStatusService.java @@ -10,6 +10,7 @@ import java.util.stream.Collectors; import javax.transaction.Transactional; +import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ViewedPagesPersistenceService; import org.apache.commons.lang3.StringUtils; import org.springframework.amqp.rabbit.core.RabbitTemplate; @@ -18,7 +19,6 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.iqser.red.service.pdftron.redaction.v1.api.model.OcrRequestMessage; import com.iqser.red.service.peristence.v1.server.configuration.MessagingConfiguration; import com.iqser.red.service.peristence.v1.server.controller.RulesController; import com.iqser.red.service.peristence.v1.server.model.NerServiceRequest; @@ -224,7 +224,7 @@ public class FileStatusService { boolean reanalyse = isReanalyse(dossier, fileStatus); - if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.imageInfoExists(dossierId, fileId)) { + if (!reanalyse && settings.isImageServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId,FileType.IMAGE_INFO)) { log.debug("Add file: {} from dossier {} to Image queue", fileId, dossierId); addToImageQueue(dossierId, fileId); return; @@ -232,7 +232,7 @@ public class FileStatusService { MessageType messageType = calculateMessageType(dossierId, fileId, reanalyse, fileStatus.getProcessingStatus()); - if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.nerEntitiesExists(dossierId, fileId)) { + if (MessageType.ANALYSE.equals(messageType) && settings.isNerServiceEnabled() && !fileManagementStorageService.typeExists(dossierId, fileId, FileType.NER_ENTITIES)) { log.debug("Add file: {} from dossier {} to NER queue", fileId, dossierId); addToNerQueue(dossierId, fileId); return; @@ -276,7 +276,8 @@ public class FileStatusService { return !fileStatus.getProcessingStatus() .equals(ProcessingStatus.UNPROCESSED) && !fileStatus.getProcessingStatus() .equals(ProcessingStatus.ANALYSE) && !fileStatus.getProcessingStatus() - .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() + .equals(ProcessingStatus.FULLREPROCESS) && fileStatus.getRulesVersion() == rulesController.getVersion(dossier.getDossierTemplateId()) + && (fileStatus.getLastFileAttributeChange() == null || fileStatus.getLastProcessed() .isAfter(fileStatus.getLastFileAttributeChange())); } @@ -284,7 +285,7 @@ public class FileStatusService { private MessageType calculateMessageType(String dossierId, String fileId, boolean reanalyse, ProcessingStatus processingStatus) { - if (!fileManagementStorageService.textExists(dossierId, fileId)) { + if (!fileManagementStorageService.typeExists(dossierId, fileId, FileType.TEXT)) { return MessageType.STRUCTURE_ANALYSE; } if (ProcessingStatus.NER_ANALYZING.equals(processingStatus)) { @@ -437,7 +438,7 @@ public class FileStatusService { private void addToOcrQueue(String dossierId, String fileId, int priority) { try { - rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new OcrRequestMessage(dossierId, fileId)), message -> { + rabbitTemplate.convertAndSend(MessagingConfiguration.OCR_QUEUE, objectMapper.writeValueAsString(new DocumentRequest(dossierId, fileId)), message -> { message.getMessageProperties().setPriority(priority); return message; }); diff --git a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java index bc0f96f54..160962e28 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java +++ b/persistence-service-v1/persistence-service-server-v1/src/main/java/com/iqser/red/service/peristence/v1/server/service/download/DownloadPreparationService.java @@ -1,13 +1,6 @@ package com.iqser.red.service.peristence.v1.server.service.download; -import java.util.Map; -import java.util.stream.Collectors; - -import javax.transaction.Transactional; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest; +import com.iqser.red.service.pdftron.redaction.v1.api.model.RedactionRequest; import com.iqser.red.service.peristence.v1.server.client.RedactionClient; import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.peristence.v1.server.service.RedactionLogService; @@ -18,11 +11,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.dossier. import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.ReportTemplateEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.download.DownloadStatusEntity; import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DownloadStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.FileStatusPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.NotificationPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.ReportTemplatePersistenceService; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.*; import com.iqser.red.service.persistence.service.v1.api.model.audit.AddNotificationRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.DownloadFileType; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; @@ -31,9 +20,13 @@ import com.iqser.red.service.redaction.report.v1.api.model.ReportResultMessage; import com.iqser.red.service.redaction.report.v1.api.model.StoredFileInformation; import com.iqser.red.service.redaction.v1.model.AnnotateRequest; import com.iqser.red.service.redaction.v1.model.RedactionLog; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import javax.transaction.Transactional; +import java.util.Map; +import java.util.stream.Collectors; @Slf4j @Service @@ -94,21 +87,17 @@ public class DownloadPreparationService { if (downloadFileType.name().equals(DownloadFileType.ORIGINAL.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Original", fileStatus.getFilename(), original)); } - if (downloadFileType.name().equals(DownloadFileType.ANNOTATED.name())) { - fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Annotated", addSuffix(fileStatus.getFilename(), "annotated"), - getAnnotated(dossier.getDossierTemplateId(), fileStatus.getDossierId(), fileId))); - } if (downloadFileType.name().equals(DownloadFileType.PREVIEW.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Preview", addSuffix(fileStatus.getFilename(), "highlighted"), - getPreview(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled()))); + getPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled()))); } if (downloadFileType.name().equals(DownloadFileType.DELTA_PREVIEW.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Delta Preview", addSuffix(fileStatus.getFilename(), "delta_highlighted"), - getDeltaPreview(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled()))); + getDeltaPreview(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkPreviewEnabled()))); } if (downloadFileType.name().equals(DownloadFileType.REDACTED.name())) { fileSystemBackedArchiver.addEntry(new FileSystemBackedArchiver.ArchiveModel("Redacted", addSuffix(fileStatus.getFilename(), "redacted"), - getRedacted(original, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkEnabled()))); + getRedacted(dossier.getId(), fileId, redactionLog, dossier.getDossierTemplateId(), dossier.isWatermarkEnabled()))); } } log.info("Successfully added file {}/{} for downloadId {}, took {}", i, fileIds @@ -195,11 +184,12 @@ public class DownloadPreparationService { } - private byte[] getRedacted(byte[] original, RedactionLog redactionLog, String dossierTemplateId, + private byte[] getRedacted(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) { - return pdfTronRedactionClient.redact(PdfTronRedactionRequest.builder() - .document(original) + return pdfTronRedactionClient.redact(RedactionRequest.builder() + .dossierId(dossierId) + .fileId(fileId) .dossierTemplateId(dossierTemplateId) .redactionLog(redactionLog) .applyWatermark(watermarkEnabled) @@ -207,10 +197,11 @@ public class DownloadPreparationService { } - private byte[] getPreview(byte[] original, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) { + private byte[] getPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) { - return pdfTronRedactionClient.redactionPreview(PdfTronRedactionRequest.builder() - .document(original) + return pdfTronRedactionClient.redactionPreview(RedactionRequest.builder() + .dossierId(dossierId) + .fileId(fileId) .dossierTemplateId(dossierTemplateId) .redactionLog(redactionLog) .applyWatermark(watermarkEnabled) @@ -218,10 +209,11 @@ public class DownloadPreparationService { } - private byte[] getDeltaPreview(byte[] original, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) { + private byte[] getDeltaPreview(String dossierId, String fileId, RedactionLog redactionLog, String dossierTemplateId, boolean watermarkEnabled) { - return pdfTronRedactionClient.redactionPreviewDiff(PdfTronRedactionRequest.builder() - .document(original) + return pdfTronRedactionClient.redactionPreviewDiff(RedactionRequest.builder() + .dossierId(dossierId) + .fileId(fileId) .dossierTemplateId(dossierTemplateId) .redactionLog(redactionLog) .applyWatermark(watermarkEnabled) diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java index aaab8505c..3d694447d 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/service/FileTesterAndProvider.java @@ -7,13 +7,14 @@ import com.iqser.red.service.peristence.v1.server.integration.client.UploadClien import com.iqser.red.service.peristence.v1.server.service.FileManagementStorageService; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.Dossier; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; import com.iqser.red.service.redaction.v1.model.SectionGrid; import lombok.SneakyThrows; +import org.apache.commons.codec.binary.Base64; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -66,8 +67,10 @@ public class FileTesterAndProvider { @SneakyThrows public String testAndProvideFileQuick(Dossier dossier, String fileName) { + var fileId = Base64.encodeBase64String((dossier.getId() + fileName).getBytes(StandardCharsets.UTF_8)); - BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), fileName, dossier.getId(), "1"); + AddFileRequest upload = new AddFileRequest(fileName, fileId, dossier.getId(), "1"); + fileManagementStorageService.storeObject(dossier.getId(), fileId, FileType.UNTOUCHED, "test".getBytes(StandardCharsets.UTF_8)); JSONPrimitive uploadResult = uploadClient.upload(upload); return uploadResult.getValue(); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java index d4738e3d0..566883c44 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/FileTest.java @@ -1,35 +1,26 @@ package com.iqser.red.service.peristence.v1.server.integration.tests; -import static org.assertj.core.api.Assertions.assertThat; - -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.util.Collections; -import java.util.List; - -import com.iqser.red.service.peristence.v1.server.integration.client.*; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; -import org.junit.Test; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; - import com.google.common.collect.Sets; +import com.iqser.red.service.peristence.v1.server.integration.client.*; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTemplateTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.DossierTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.service.TypeProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AddRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ForceRedactionRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImageRecategorizationRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.LegalBasisChangeRequest; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.RemoveRedactionRequest; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.CreateOrUpdateDossierRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.WorkflowStatus; +import org.junit.Test; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; -import feign.FeignException; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; public class FileTest extends AbstractPersistenceServerServiceTest { @@ -60,20 +51,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { @Autowired private ViewedPagesClient viewedPagesClient; - @Test - public void testEmptyFile() { - - var dossier = dossierTesterAndProvider.provideTestDossier(); - BinaryFileRequest upload = new BinaryFileRequest("".getBytes(StandardCharsets.UTF_8), "emptyFile.pdf", dossier.getId(), "1"); - - try { - uploadClient.upload(upload); - } catch (FeignException e) { - assertThat(e.status()).isEqualTo(400); - } - } - - @Test public void testFileSoftDeleteReupload() { @@ -122,7 +99,7 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(viewedPages.size()).isEqualTo(1); - BinaryFileRequest upload = new BinaryFileRequest("test".getBytes(StandardCharsets.UTF_8), filename, dossier.getId(), "1"); + AddFileRequest upload = new AddFileRequest(filename, file.getId(), dossier.getId(), "1"); JSONPrimitive uploadResult = uploadClient.upload(upload); loadedFile = fileClient.getFileStatus(dossier.getId(), uploadResult.getValue()); @@ -229,13 +206,6 @@ public class FileTest extends AbstractPersistenceServerServiceTest { assertThat(loadedFile.getLastApprover()).isEqualTo("2"); - assertThat(uploadClient.getOriginal(dossier.getId(), file.getId())).isNotNull(); - assertThat(uploadClient.getRedacted(dossier.getId(), file.getId())).isNotNull(); - assertThat(uploadClient.getFlatRedacted(dossier.getId(), file.getId())).isNotNull(); - assertThat(uploadClient.getPreview(dossier.getId(), file.getId())).isNotNull(); - assertThat(uploadClient.getAnnotated(dossier.getId(), file.getId())).isNotNull(); - - fileClient.setStatusUnderReview(dossier.getId(), file.getId(), null); diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java index d14eafc99..600cb5f6e 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/ReanalysisTest.java @@ -7,20 +7,18 @@ import com.iqser.red.service.peristence.v1.server.integration.service.DossierTes import com.iqser.red.service.peristence.v1.server.integration.service.FileTesterAndProvider; import com.iqser.red.service.peristence.v1.server.integration.utils.AbstractPersistenceServerServiceTest; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.FileRepository; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.BinaryFileRequest; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.AddFileRequest; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.ProcessingStatus; import feign.FeignException; import org.junit.Test; -import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import java.nio.charset.StandardCharsets; import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; public class ReanalysisTest extends AbstractPersistenceServerServiceTest { @@ -74,30 +72,6 @@ public class ReanalysisTest extends AbstractPersistenceServerServiceTest { } - @Test - public void testImportRedactions() { - - var dossier = dossierTesterAndProvider.provideTestDossier(); - - var file = fileTesterAndProvider.testAndProvideFile(dossier); - BinaryFileRequest upload = new BinaryFileRequest("testing".getBytes(StandardCharsets.UTF_8), "emptyFile.pdf", dossier.getId(), "1"); - - DocumentRequest documentRequest = DocumentRequest.builder() - .document(upload.getData()) - .fileId(file.getId()).build(); - - when(pdfTronRedactionClient.optimize(Mockito.any())).thenAnswer((args) -> - PdfTronOptimizeResponse.builder() - .document(((PdfTronOptimizeRequest) args.getArguments()[0]).getDocument()) - .numberOfPages(100).build()); - - try { - reanalysisClient.importRedactions(documentRequest); - } catch (FeignException e) { - assertThat(e.status()).isEqualTo(400); - } - } - private void resetProcessingStatus(FileModel file) { diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java index 9356133a9..2798620fa 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/utils/AbstractPersistenceServerServiceTest.java @@ -5,6 +5,8 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.when; +import com.iqser.red.service.pdftron.redaction.v1.api.model.ByteContentDocument; +import com.iqser.red.service.pdftron.redaction.v1.api.model.UntouchedDocumentResponse; import com.iqser.red.service.persistence.management.v1.processor.service.persistence.repository.*; import org.assertj.core.util.Lists; import org.junit.After; @@ -31,10 +33,6 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import com.iqser.red.service.pdftron.redaction.v1.api.model.DocumentRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronFlattenResponse; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionRequest; -import com.iqser.red.service.pdftron.redaction.v1.api.model.PdfTronRedactionResult; import com.iqser.red.service.peristence.v1.server.Application; import com.iqser.red.service.peristence.v1.server.client.RedactionClient; import com.iqser.red.service.peristence.v1.server.client.SearchClient; @@ -48,6 +46,7 @@ import com.iqser.red.service.redaction.v1.model.RedactionLog; import com.iqser.red.service.redaction.v1.model.RedactionResult; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; @RunWith(SpringRunner.class) @EnableFeignClients(basePackageClasses = FileClient.class) @@ -149,19 +148,38 @@ public abstract class AbstractPersistenceServerServiceTest { Object[] args = answer.getArguments(); DocumentRequest d = (DocumentRequest) args[0]; - storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), d.getDocument()); - return null; - }).when(pdfTronRedactionClient).saveOptimizedAndAnnotations(any()); + var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.UNTOUCHED); + + var untouched = storageService.getObject(untouchedObjectId); + storageService.storeObject(StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN), IOUtils.toByteArray(untouched.getInputStream())); + return UntouchedDocumentResponse.builder().build(); + + }).when(pdfTronRedactionClient).processUntouchedDocument(any()); - when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((args) -> - new PdfTronRedactionResult(((PdfTronRedactionRequest) args.getArguments()[0]).getDocument())); - when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((args) -> - new PdfTronRedactionResult(((PdfTronRedactionRequest) args.getArguments()[0]).getDocument())); + when(pdfTronRedactionClient.redact(Mockito.any())).thenAnswer((answer) ->{ + Object[] args = answer.getArguments(); + DocumentRequest d = (DocumentRequest) args[0]; + var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN); + + return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream())); + }); + when(pdfTronRedactionClient.redactionPreview(Mockito.any())).thenAnswer((answer) ->{ + Object[] args = answer.getArguments(); + DocumentRequest d = (DocumentRequest) args[0]; + var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN); + + return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream())); + }); + when(pdfTronRedactionClient.redactionPreviewDiff(Mockito.any())).thenAnswer((answer) ->{ + Object[] args = answer.getArguments(); + DocumentRequest d = (DocumentRequest) args[0]; + var untouchedObjectId = StorageIdUtils.getStorageId(d.getDossierId(), d.getFileId(), FileType.ORIGIN); + + return new ByteContentDocument(IOUtils.toByteArray(storageService.getObject(untouchedObjectId).getInputStream())); + }); - when(pdfTronRedactionClient.flatDocument(Mockito.any())).thenAnswer((args) -> - new PdfTronFlattenResponse(((PdfTronFlattenRequest) args.getArguments()[0]).getDocument())); when(redactionClient.annotate(Mockito.any())).thenReturn(new AnnotateResponse("document".getBytes())); when(redactionClient.sections(Mockito.any())).thenReturn(new RedactionResult()); diff --git a/persistence-service-v1/pom.xml b/persistence-service-v1/pom.xml index 6bae450ca..6e2558758 100755 --- a/persistence-service-v1/pom.xml +++ b/persistence-service-v1/pom.xml @@ -27,7 +27,7 @@ 3.108.0 2.36.0 - 3.75.0 + 3.79.0 3.45.0