From 4f9c5626b5736fd2a52bd08f30bae9a55a51731d Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Fri, 11 Mar 2022 10:23:14 +0100 Subject: [PATCH] file extensions are now assumed to be part of the request --- config.yaml | 2 -- pyinfra/test/unit_tests/queue_visitor_test.py | 2 +- pyinfra/visitor.py | 14 ++++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config.yaml b/config.yaml index b17a693..ebbd7a5 100755 --- a/config.yaml +++ b/config.yaml @@ -38,8 +38,6 @@ storage: backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure} bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from # TODO: Caller should specify exact file name, including extension! - target_file_extension: $TARGET_FILE_EXTENSION|"ORIGIN.pdf.gz" # {.TEXT.json.gz | .ORIGIN.pdf.gz} Defines type of file to pull from storage - response_file_extension: $RESPONSE_FILE_EXTENSION|"IMAGE_INFO.json.gz" s3: endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000" diff --git a/pyinfra/test/unit_tests/queue_visitor_test.py b/pyinfra/test/unit_tests/queue_visitor_test.py index 97f20f7..e6fd109 100644 --- a/pyinfra/test/unit_tests/queue_visitor_test.py +++ b/pyinfra/test/unit_tests/queue_visitor_test.py @@ -12,7 +12,7 @@ def visitor(storage, analysis_callback, response_strategy): @pytest.fixture() def body(): - return {"dossierId": "folder", "fileId": "file"} + return {"dossierId": "folder", "fileId": "file", "targetFileExtension": "in", "responseFileExtension": "out"} @pytest.mark.parametrize("client_name", ["mock", "azure", "s3"], scope="session") diff --git a/pyinfra/visitor.py b/pyinfra/visitor.py index af298c2..defe94d 100644 --- a/pyinfra/visitor.py +++ b/pyinfra/visitor.py @@ -8,21 +8,20 @@ from pyinfra.storage.storage import Storage def get_object_name(body): - dossier_id, file_id = itemgetter("dossierId", "fileId")(body) - object_name = f"{dossier_id}/{file_id}.{CONFIG.target_file_extension}" + dossier_id, file_id, target_file_extension = itemgetter("dossierId", "fileId", "targetFileExtension")(body) + object_name = f"{dossier_id}/{file_id}.{target_file_extension}" return object_name def get_response_object_name(body): - dossier_id, file_id = itemgetter("dossierId", "fileId")(body) - object_name = f"{dossier_id}/{file_id}.{CONFIG.response_file_extension}" + dossier_id, file_id, response_file_extension = itemgetter("dossierId", "fileId", "responseFileExtension")(body) + object_name = f"{dossier_id}/{file_id}.{response_file_extension}" return object_name def get_object_descriptor(body): return { "bucket_name": CONFIG.storage.bucket, - # TODO: send complete filename in request, as discussed! "object_name": get_object_name(body) } @@ -30,7 +29,6 @@ def get_object_descriptor(body): def get_response_object_descriptor(body): return { "bucket_name": CONFIG.storage.bucket, - # TODO: send complete filename in request, as discussed! "object_name": get_response_object_name(body) } @@ -78,3 +76,7 @@ class QueueVisitor: def __call__(self, body): result_body = self.load_and_process(body) return self.response_strategy(result_body) + + +# TODO: 1) add compression / decompression to visitor +# 2) correct keys and values for storage response