refactoring; removed obsolete config entries and code

config.yaml

@@ -1,13 +1,9 @@
 service:
   logging_level: $LOGGING_LEVEL_ROOT|DEBUG  # Logging level for service logger
   name: $SERVICE_NAME|research  # Default service name for research service, used for prometheus metric name
-#  target_file_extension: $TARGET_FILE_EXTENSION|"json.gz"  # Extension for files to download from storage and process
 
-  # TODO: will become obsolete by below changes
-  response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz"  # Extension for response files to upload to storage
   # Specifies, how to handle the `page` key of a request. "multi" will download all pages  matching the list of pages
   # specified in the request
-  download_strategy: $DOWNLOAD_STRATEGY|multi
   response_formatter: default  # TODO: write formatter for analysis tasks that pulls metadata content into root of response json
   # Note: This is not really the right place for this. It should be configured on a per-service basis.
   operations:

pyinfra/visitor/strategies/response/aggregation.py

@@ -1,4 +1,3 @@
-import json
 from collections import deque
 from typing import Callable
 

pyinfra/visitor/strategies/response/storage.py

@@ -7,8 +7,9 @@ from pyinfra.visitor.strategies.response.response import ResponseStrategy
 
 
 class StorageStrategy(ResponseStrategy):
-    def __init__(self, storage):
+    def __init__(self, storage, response_file_extension="out"):
         self.storage = storage
+        self.response_file_extension = response_file_extension
 
     def handle_response(self, body: dict):
         response_object_descriptor = self.get_response_object_descriptor(body)
@@ -23,11 +24,10 @@ class StorageStrategy(ResponseStrategy):
             "object_name": self.get_response_object_name(body),
         }
 
-    @staticmethod
+    def get_response_object_name(self, body):
-    def get_response_object_name(body):
 
         dossier_id, file_id = itemgetter("dossierId", "fileId")(body)
 
-        object_name = f"{dossier_id}/{file_id}/.{CONFIG.service.response_file_extension}"
+        object_name = f"{dossier_id}/{file_id}/.{self.response_file_extension}"
 
         return object_name

scripts/mock_client.py

@@ -96,9 +96,6 @@ def main(args):
     channel = make_channel(connection)
     declare_queue(channel, CONFIG.rabbitmq.queues.input)
     declare_queue(channel, CONFIG.rabbitmq.queues.output)
-    if args.analysis_container == "table_parsing":
-        CONFIG["service"]["target_file_extension"] = "json.gz"
-        # CONFIG["service"]["download_strategy"] = "multi"
 
     for body in build_message_bodies(args.analysis_container, args.bucket_name):
         channel.basic_publish("", CONFIG.rabbitmq.queues.input, body)

test/conftest.py

@@ -218,6 +218,5 @@ def file_descriptor_manager(component_factory):
 @pytest.fixture
 def component_factory():
     MAIN_CONFIG["service"]["operations"] = TEST_CONFIG.service.operations
-    MAIN_CONFIG["service"]["download_strategy"] = "single"
 
     return get_component_factory(MAIN_CONFIG)

test/integration_tests/serve_test.py

@@ -205,11 +205,10 @@ def components_type(request):
 
 
 @pytest.fixture
-def real_components(url, download_strategy_type):
+def real_components(url):
 
     CONFIG["service"]["operations"] = TEST_CONFIG.service.operations
     CONFIG["service"]["response_formatter"] = TEST_CONFIG.service.response_formatter
-    CONFIG["service"]["download_strategy"] = download_strategy_type
 
     component_factory = get_component_factory(CONFIG)
 
@@ -223,24 +222,17 @@ def real_components(url, download_strategy_type):
 
 
 @pytest.fixture
-def download_strategy_type(many_to_n):
+def test_components(url, queue_manager, storage):
-    return "multi" if many_to_n else "single"
-
-
-@pytest.fixture
-def test_components(url, queue_manager, storage, download_strategy_type):
     pass
     #
     # component_factory = ComponentFactory(CONFIG)
     #
-    # download_strategy = component_factory.get_download_strategy(download_strategy_type)
     # file_descriptor_manager = component_factory.get_file_descriptor_manager()
     #
     # visitor = QueueVisitor(
     #     storage=storage,
     #     callback=component_factory.get_callback(url),
     #     response_strategy=component_factory.get_response_strategy(storage),
-    #     download_strategy=download_strategy,
     # )
     # consumer = Consumer(visitor, queue_manager)
     #