Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

added logic for uploading response files to a folder, which defaults to the name of the operation used

Browse Source
This commit is contained in:
Matthias Bisping 2022-06-10 13:03:40 +02:00
parent 13b6388e5a
commit 3a3c497383
2 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.yaml
View File

@ -1,5 +1,7 @@
service:
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger
response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for files uploaded to storage
response_folder: $RESPONSE_FOLDER|null
probing_webserver:
host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address

20
pyinfra/visitor.py
View File

@ -30,8 +30,8 @@ def unique_hash(pages):
def get_object_name(body):
dossier_id, file_id, target_file_extension = itemgetter("dossierId", "fileId", "targetFileExtension")(body)
object_name = f"{dossier_id}/{file_id}.{target_file_extension}"
dossier_id, file_id = itemgetter("dossierId", "fileId")(body)
object_name = f"{dossier_id}/{file_id}.{CONFIG.service.response_file_extension}"
return object_name
@ -47,7 +47,7 @@ def get_response_object_name(body):
"dossierId", "fileId", "pages", "id", "responseFileExtension"
)(body)
object_name = f"{dossier_id}/{file_id}_{unique_hash(pages)}-id:{idnt}.{response_file_extension}"
object_name = f"{dossier_id}/{file_id}/{unique_hash(pages)}-id:{idnt}.{response_file_extension}"
return object_name
@ -141,7 +141,7 @@ class AggregationStorageStrategy(ResponseStrategy):
analysis_payload : {data: ..., metadata: ...}
"""
storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)}
storage_upload_info = build_storage_upload_info(analysis_payload, request_metadata)
if analysis_payload["data"]:
return self.put_object(json.dumps(analysis_payload).encode(), storage_upload_info)
@ -158,6 +158,18 @@ class AggregationStorageStrategy(ResponseStrategy):
yield self.upload_or_aggregate(analysis_payload, request_metadata, last=not result_data.peek(False))
def build_storage_upload_info(analysis_payload, request_metadata):
storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)}
storage_upload_info["fileId"] = build_file_path(
storage_upload_info, storage_upload_info.get("operation", CONFIG.service.response_folder)
)
return storage_upload_info
def build_file_path(storage_upload_info, folder):
return f"{storage_upload_info['fileId']}" + (f"/{folder}" if folder else "")
class InvalidStorageItemFormat(ValueError):
pass