diff --git a/config.yaml b/config.yaml index 9586ace..be69596 100755 --- a/config.yaml +++ b/config.yaml @@ -2,8 +2,8 @@ service: logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger response: save: True # file-to-storage upload - extension: ".NER_ENTITIES.json.gz" - # extension: ".OBJECTS.json.gz" + #extension: $RESPONSE_FILE_EXTENSION|".NER_ENTITIES.json.gz" # {.OBJECTS.json.gz | .NER_ENTITIES.json.gz} + extension: $RESPONSE_FILE_EXTENSION|".OBJECTS.json.gz" probing_webserver: host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address @@ -37,7 +37,8 @@ storage: backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure} bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from - target_file_extension: $TARGET_FILE_EXTENSION|pdf # Defines type of file to pull from storage + #target_file_extension: $TARGET_FILE_EXTENSION|".TEXT.json.gz" # {.TEXT.json.gz | .ORIGIN.pdf.gz} Defines type of file to pull from storage + target_file_extension: $TARGET_FILE_EXTENSION|".ORIGIN.pdf.gz" s3: endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000" diff --git a/pyinfra/core.py b/pyinfra/core.py index e53cea3..9a6c82f 100644 --- a/pyinfra/core.py +++ b/pyinfra/core.py @@ -5,15 +5,17 @@ from operator import itemgetter import requests -from pyinfra.callback import json_wrap +from pyinfra.config import CONFIG from pyinfra.exceptions import DataLoadingFailure, AnalysisFailure, ProcessingFailure -from pyinfra.utils.file import dossier_id_and_file_id_to_compressed_storage_pdf_object_name +from pyinfra.utils.file import combine_dossier_id_and_file_id_and_extension def make_storage_data_loader(storage, bucket_name): def get_object_name(payload: dict) -> str: dossier_id, file_id = itemgetter("dossierId", "fileId")(payload) - object_name = dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, file_id) + object_name = combine_dossier_id_and_file_id_and_extension( + dossier_id, file_id, CONFIG.storage.target_file_extension + ) return object_name def download(payload): diff --git a/pyinfra/utils/file.py b/pyinfra/utils/file.py index 2d33f35..a2cbbd2 100644 --- a/pyinfra/utils/file.py +++ b/pyinfra/utils/file.py @@ -6,19 +6,11 @@ import os from pyinfra.config import CONFIG -def produce_compressed_storage_pdf_object_name(path_no_ext, ext=None): - if not ext: - ext = CONFIG.storage.target_file_extension - return f"{path_no_ext}.ORIGIN.{ext}.gz" - - -def dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, file_id): - path_no_ext = os.path.join(dossier_id, file_id) - pdf_object_name = produce_compressed_storage_pdf_object_name(path_no_ext) - return pdf_object_name +def combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, extension): + return f"{dossier_id}/{file_id}{extension}" def upload_compressed_response(storage, bucket_name, dossier_id, file_id, result) -> None: data = gzip.compress(result.encode()) - path_gz = f"{dossier_id}/{file_id}{CONFIG.service.response.extension}" + path_gz = combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, CONFIG.service.response.extension) storage.put_object(bucket_name, path_gz, data) diff --git a/scripts/manage_minio.py b/scripts/manage_minio.py index d141b09..c1bbc65 100644 --- a/scripts/manage_minio.py +++ b/scripts/manage_minio.py @@ -7,7 +7,7 @@ from tqdm import tqdm from pyinfra.config import CONFIG from pyinfra.storage.storages import get_s3_storage -from pyinfra.utils.file import dossier_id_and_file_id_to_compressed_storage_pdf_object_name +from pyinfra.utils.file import combine_dossier_id_and_file_id_and_extension def parse_args(): @@ -29,7 +29,9 @@ def parse_args(): def add_file_compressed(storage, bucket_name, dossier_id, path) -> None: - path_gz = dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, Path(path).stem) + path_gz = combine_dossier_id_and_file_id_and_extension( + dossier_id, Path(path).stem, CONFIG.storage.target_file_extension + ) with open(path, "rb") as f: data = gzip.compress(f.read())