response save on storage logic added

config.yaml

@@ -2,8 +2,8 @@ service:
   logging_level: $LOGGING_LEVEL_ROOT|DEBUG  # Logging level for service logger
   response:
     save: True  # file-to-storage upload
-    extension: ".NER_ENTITIES.json.gz"
-    # extension: ".OBJECTS.json.gz"
+    #extension: $RESPONSE_FILE_EXTENSION|".NER_ENTITIES.json.gz"  # {.OBJECTS.json.gz | .NER_ENTITIES.json.gz}
+    extension: $RESPONSE_FILE_EXTENSION|".OBJECTS.json.gz"
 
 probing_webserver:
   host: $PROBING_WEBSERVER_HOST|"0.0.0.0"  # Probe webserver address
@@ -37,7 +37,8 @@ storage:
 
   backend: $STORAGE_BACKEND|s3  # The type of storage to use {s3, azure}
   bucket: $STORAGE_BUCKET|"pyinfra-test-bucket"  # The bucket / container to pull files specified in queue requests from
-  target_file_extension: $TARGET_FILE_EXTENSION|pdf  # Defines type of file to pull from storage
+  #target_file_extension: $TARGET_FILE_EXTENSION|".TEXT.json.gz"  # {.TEXT.json.gz | .ORIGIN.pdf.gz} Defines type of file to pull from storage
+  target_file_extension: $TARGET_FILE_EXTENSION|".ORIGIN.pdf.gz"
 
   s3:
     endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000"

pyinfra/core.py

@@ -5,15 +5,17 @@ from operator import itemgetter
 
 import requests
 
-from pyinfra.callback import json_wrap
+from pyinfra.config import CONFIG
 from pyinfra.exceptions import DataLoadingFailure, AnalysisFailure, ProcessingFailure
-from pyinfra.utils.file import dossier_id_and_file_id_to_compressed_storage_pdf_object_name
+from pyinfra.utils.file import combine_dossier_id_and_file_id_and_extension
 
 
 def make_storage_data_loader(storage, bucket_name):
     def get_object_name(payload: dict) -> str:
         dossier_id, file_id = itemgetter("dossierId", "fileId")(payload)
-        object_name = dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, file_id)
+        object_name = combine_dossier_id_and_file_id_and_extension(
+            dossier_id, file_id, CONFIG.storage.target_file_extension
+        )
         return object_name
 
     def download(payload):

pyinfra/utils/file.py

@@ -6,19 +6,11 @@ import os
 from pyinfra.config import CONFIG
 
 
-def produce_compressed_storage_pdf_object_name(path_no_ext, ext=None):
-    if not ext:
-        ext = CONFIG.storage.target_file_extension
-    return f"{path_no_ext}.ORIGIN.{ext}.gz"
-
-
-def dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, file_id):
-    path_no_ext = os.path.join(dossier_id, file_id)
-    pdf_object_name = produce_compressed_storage_pdf_object_name(path_no_ext)
-    return pdf_object_name
+def combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, extension):
+    return f"{dossier_id}/{file_id}{extension}"
 
 
 def upload_compressed_response(storage, bucket_name, dossier_id, file_id, result) -> None:
     data = gzip.compress(result.encode())
-    path_gz = f"{dossier_id}/{file_id}{CONFIG.service.response.extension}"
+    path_gz = combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, CONFIG.service.response.extension)
     storage.put_object(bucket_name, path_gz, data)

scripts/manage_minio.py

@@ -7,7 +7,7 @@ from tqdm import tqdm
 
 from pyinfra.config import CONFIG
 from pyinfra.storage.storages import get_s3_storage
-from pyinfra.utils.file import dossier_id_and_file_id_to_compressed_storage_pdf_object_name
+from pyinfra.utils.file import combine_dossier_id_and_file_id_and_extension
 
 
 def parse_args():
@@ -29,7 +29,9 @@ def parse_args():
 
 def add_file_compressed(storage, bucket_name, dossier_id, path) -> None:
 
-    path_gz = dossier_id_and_file_id_to_compressed_storage_pdf_object_name(dossier_id, Path(path).stem)
+    path_gz = combine_dossier_id_and_file_id_and_extension(
+        dossier_id, Path(path).stem, CONFIG.storage.target_file_extension
+    )
 
     with open(path, "rb") as f:
         data = gzip.compress(f.read())