From 3a3c49738350875251ac0f5f033752cdfaf8e9bd Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Fri, 10 Jun 2022 13:03:40 +0200 Subject: [PATCH] added logic for uploading response files to a folder, which defaults to the name of the operation used --- config.yaml | 2 ++ pyinfra/visitor.py | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/config.yaml b/config.yaml index a7f8efe..a2d20a0 100755 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,7 @@ service: logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger + response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for files uploaded to storage + response_folder: $RESPONSE_FOLDER|null probing_webserver: host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address diff --git a/pyinfra/visitor.py b/pyinfra/visitor.py index 026eed4..fda3dbb 100644 --- a/pyinfra/visitor.py +++ b/pyinfra/visitor.py @@ -30,8 +30,8 @@ def unique_hash(pages): def get_object_name(body): - dossier_id, file_id, target_file_extension = itemgetter("dossierId", "fileId", "targetFileExtension")(body) - object_name = f"{dossier_id}/{file_id}.{target_file_extension}" + dossier_id, file_id = itemgetter("dossierId", "fileId")(body) + object_name = f"{dossier_id}/{file_id}.{CONFIG.service.response_file_extension}" return object_name @@ -47,7 +47,7 @@ def get_response_object_name(body): "dossierId", "fileId", "pages", "id", "responseFileExtension" )(body) - object_name = f"{dossier_id}/{file_id}_{unique_hash(pages)}-id:{idnt}.{response_file_extension}" + object_name = f"{dossier_id}/{file_id}/{unique_hash(pages)}-id:{idnt}.{response_file_extension}" return object_name @@ -141,7 +141,7 @@ class AggregationStorageStrategy(ResponseStrategy): analysis_payload : {data: ..., metadata: ...} """ - storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)} + storage_upload_info = build_storage_upload_info(analysis_payload, request_metadata) if analysis_payload["data"]: return self.put_object(json.dumps(analysis_payload).encode(), storage_upload_info) @@ -158,6 +158,18 @@ class AggregationStorageStrategy(ResponseStrategy): yield self.upload_or_aggregate(analysis_payload, request_metadata, last=not result_data.peek(False)) +def build_storage_upload_info(analysis_payload, request_metadata): + storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)} + storage_upload_info["fileId"] = build_file_path( + storage_upload_info, storage_upload_info.get("operation", CONFIG.service.response_folder) + ) + return storage_upload_info + + +def build_file_path(storage_upload_info, folder): + return f"{storage_upload_info['fileId']}" + (f"/{folder}" if folder else "") + + class InvalidStorageItemFormat(ValueError): pass