Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

file extensions are now assumed to be part of the request

Browse Source
This commit is contained in:
Matthias Bisping 2022-03-11 10:23:14 +01:00
parent 91980ff3ed
commit 4f9c5626b5
3 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.yaml
View File

@ -38,8 +38,6 @@ storage:
backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure}
bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from
# TODO: Caller should specify exact file name, including extension!
target_file_extension: $TARGET_FILE_EXTENSION|"ORIGIN.pdf.gz" # {.TEXT.json.gz | .ORIGIN.pdf.gz} Defines type of file to pull from storage
response_file_extension: $RESPONSE_FILE_EXTENSION|"IMAGE_INFO.json.gz"
s3:
endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000"

2
pyinfra/test/unit_tests/queue_visitor_test.py
View File

@ -12,7 +12,7 @@ def visitor(storage, analysis_callback, response_strategy):
@pytest.fixture()
def body():
return {"dossierId": "folder", "fileId": "file"}
return {"dossierId": "folder", "fileId": "file", "targetFileExtension": "in", "responseFileExtension": "out"}
@pytest.mark.parametrize("client_name", ["mock", "azure", "s3"], scope="session")

14
pyinfra/visitor.py
View File

@ -8,21 +8,20 @@ from pyinfra.storage.storage import Storage
def get_object_name(body):
dossier_id, file_id = itemgetter("dossierId", "fileId")(body)
object_name = f"{dossier_id}/{file_id}.{CONFIG.target_file_extension}"
dossier_id, file_id, target_file_extension = itemgetter("dossierId", "fileId", "targetFileExtension")(body)
object_name = f"{dossier_id}/{file_id}.{target_file_extension}"
return object_name
def get_response_object_name(body):
dossier_id, file_id = itemgetter("dossierId", "fileId")(body)
object_name = f"{dossier_id}/{file_id}.{CONFIG.response_file_extension}"
dossier_id, file_id, response_file_extension = itemgetter("dossierId", "fileId", "responseFileExtension")(body)
object_name = f"{dossier_id}/{file_id}.{response_file_extension}"
return object_name
def get_object_descriptor(body):
return {
"bucket_name": CONFIG.storage.bucket,
# TODO: send complete filename in request, as discussed!
"object_name": get_object_name(body)
}
@ -30,7 +29,6 @@ def get_object_descriptor(body):
def get_response_object_descriptor(body):
return {
"bucket_name": CONFIG.storage.bucket,
# TODO: send complete filename in request, as discussed!
"object_name": get_response_object_name(body)
}
@ -78,3 +76,7 @@ class QueueVisitor:
def __call__(self, body):
result_body = self.load_and_process(body)
return self.response_strategy(result_body)
# TODO: 1) add compression / decompression to visitor
# 2) correct keys and values for storage response