cleaned up file descr mngr

config.yaml

@@ -5,9 +5,6 @@ service:
 
   # TODO: will become obsolete by below changes
   response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz"  # Extension for response files to upload to storage
-  # In case a service is not supposed to place response files under the root folder `dossierId/fileId` and requests do
-  # not specify an operation, this parameter can specify an output folder
-  response_folder: $RESPONSE_FOLDER|null  # FIXME: covered by operation mapping?
   # Specifies, how to handle the `page` key of a request. "multi" will download all pages  matching the list of pages
   # specified in the request
   download_strategy: $DOWNLOAD_STRATEGY|multi

pyinfra/file_descriptor_manager.py

@@ -1,11 +1,8 @@
-import json
 import os
 from _operator import itemgetter
 
 from funcy import project
 
-from pyinfra.config import CONFIG
-
 
 class FileDescriptorManager:
     def __init__(self, bucket_name, operation2file_patterns: dict = None):
@@ -24,18 +21,12 @@ class FileDescriptorManager:
 
     def get_object_name(self, queue_item_body: dict, end):
 
-        # TODO: refactor
+        file_descriptor = self.build_file_descriptor(queue_item_body, end=end)
+        file_descriptor["pages"] = [queue_item_body.get("id", 0)]
 
-        if end == "output":
-            return self.get_response_object_name(queue_item_body)
-        else:
+        object_name = self.__build_matcher(file_descriptor)
 
-            file_descriptor = self.build_file_descriptor(queue_item_body, end=end)
-            file_descriptor["pages"] = [queue_item_body.get("id", 0)]
-
-            object_name = self.__build_matcher(file_descriptor)
-
-            return object_name
+        return object_name
 
     @staticmethod
     def __build_matcher(file_descriptor):
@@ -74,7 +65,7 @@ class FileDescriptorManager:
         return self.build_matcher(queue_item_body, end="input")
 
     def build_output_matcher(self, queue_item_body):
-        return self.get_response_object_name(queue_item_body)
+        return self.build_matcher(queue_item_body, end="output")
 
     def build_matcher(self, queue_item_body, end):
         file_descriptor = self.build_file_descriptor(queue_item_body, end=end)
@@ -96,16 +87,3 @@ class FileDescriptorManager:
     def build_storage_upload_info(analysis_payload, request_metadata):
         storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)}
         return storage_upload_info
-
-    @staticmethod
-    def get_response_object_name(body):
-        # TODO: refactor
-
-        if "id" not in body:
-            body["id"] = 0
-
-        dossier_id, file_id, idnt = itemgetter("dossierId", "fileId", "id")(body)
-
-        object_name = f"{dossier_id}/{file_id}/id:{idnt}.{CONFIG.service.response_file_extension}"
-
-        return object_name

pyinfra/visitor/strategies/response/aggregation.py

@@ -38,11 +38,6 @@ class AggregationStorageStrategy(ResponseStrategy):
         """analysis_payload : {data: ..., metadata: ...}"""
         storage_upload_info = self.file_descriptor_manager.build_storage_upload_info(analysis_payload, request_metadata)
         object_descriptor = self.file_descriptor_manager.get_output_object_descriptor(storage_upload_info)
-        print(111111111)
-        print(json.dumps(storage_upload_info, indent=2))
-        print(222222222)
-        print(json.dumps(object_descriptor, indent=2))
-        print()
 
         self.add_analysis_payload_to_buffer(analysis_payload)
 

test/config.yaml

@@ -13,7 +13,7 @@ service:
         subdir: ""
         extension: IN.gz
       output:
-        subdir: ""
+        subdir: op_outp_files
         extension: OUT.gz
     default:
       input:

test/integration_tests/serve_test.py

@@ -221,8 +221,6 @@ def real_components(url, download_strategy_type):
     storage = component_factory.get_storage()
     file_descriptor_manager = component_factory.get_file_descriptor_manager()
 
-    print(download_strategy_type, component_factory.get_download_strategy())
-
     return storage, queue_manager, consumer, file_descriptor_manager