diff --git a/config.yaml b/config.yaml index eb37902..a574e3e 100755 --- a/config.yaml +++ b/config.yaml @@ -1,13 +1,9 @@ service: logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger name: $SERVICE_NAME|research # Default service name for research service, used for prometheus metric name -# target_file_extension: $TARGET_FILE_EXTENSION|"json.gz" # Extension for files to download from storage and process - # TODO: will become obsolete by below changes - response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for response files to upload to storage # Specifies, how to handle the `page` key of a request. "multi" will download all pages matching the list of pages # specified in the request - download_strategy: $DOWNLOAD_STRATEGY|multi response_formatter: default # TODO: write formatter for analysis tasks that pulls metadata content into root of response json # Note: This is not really the right place for this. It should be configured on a per-service basis. operations: diff --git a/pyinfra/visitor/strategies/response/aggregation.py b/pyinfra/visitor/strategies/response/aggregation.py index 32713cb..a8e4838 100644 --- a/pyinfra/visitor/strategies/response/aggregation.py +++ b/pyinfra/visitor/strategies/response/aggregation.py @@ -1,4 +1,3 @@ -import json from collections import deque from typing import Callable diff --git a/pyinfra/visitor/strategies/response/storage.py b/pyinfra/visitor/strategies/response/storage.py index fa4e6c9..77326ec 100644 --- a/pyinfra/visitor/strategies/response/storage.py +++ b/pyinfra/visitor/strategies/response/storage.py @@ -7,8 +7,9 @@ from pyinfra.visitor.strategies.response.response import ResponseStrategy class StorageStrategy(ResponseStrategy): - def __init__(self, storage): + def __init__(self, storage, response_file_extension="out"): self.storage = storage + self.response_file_extension = response_file_extension def handle_response(self, body: dict): response_object_descriptor = self.get_response_object_descriptor(body) @@ -23,11 +24,10 @@ class StorageStrategy(ResponseStrategy): "object_name": self.get_response_object_name(body), } - @staticmethod - def get_response_object_name(body): + def get_response_object_name(self, body): dossier_id, file_id = itemgetter("dossierId", "fileId")(body) - object_name = f"{dossier_id}/{file_id}/.{CONFIG.service.response_file_extension}" + object_name = f"{dossier_id}/{file_id}/.{self.response_file_extension}" return object_name diff --git a/scripts/mock_client.py b/scripts/mock_client.py index c38b1f0..246333a 100644 --- a/scripts/mock_client.py +++ b/scripts/mock_client.py @@ -96,9 +96,6 @@ def main(args): channel = make_channel(connection) declare_queue(channel, CONFIG.rabbitmq.queues.input) declare_queue(channel, CONFIG.rabbitmq.queues.output) - if args.analysis_container == "table_parsing": - CONFIG["service"]["target_file_extension"] = "json.gz" - # CONFIG["service"]["download_strategy"] = "multi" for body in build_message_bodies(args.analysis_container, args.bucket_name): channel.basic_publish("", CONFIG.rabbitmq.queues.input, body) diff --git a/test/conftest.py b/test/conftest.py index 7805a84..43e3806 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -218,6 +218,5 @@ def file_descriptor_manager(component_factory): @pytest.fixture def component_factory(): MAIN_CONFIG["service"]["operations"] = TEST_CONFIG.service.operations - MAIN_CONFIG["service"]["download_strategy"] = "single" return get_component_factory(MAIN_CONFIG) diff --git a/test/integration_tests/serve_test.py b/test/integration_tests/serve_test.py index 7d6f97c..dc5c8af 100644 --- a/test/integration_tests/serve_test.py +++ b/test/integration_tests/serve_test.py @@ -205,11 +205,10 @@ def components_type(request): @pytest.fixture -def real_components(url, download_strategy_type): +def real_components(url): CONFIG["service"]["operations"] = TEST_CONFIG.service.operations CONFIG["service"]["response_formatter"] = TEST_CONFIG.service.response_formatter - CONFIG["service"]["download_strategy"] = download_strategy_type component_factory = get_component_factory(CONFIG) @@ -223,24 +222,17 @@ def real_components(url, download_strategy_type): @pytest.fixture -def download_strategy_type(many_to_n): - return "multi" if many_to_n else "single" - - -@pytest.fixture -def test_components(url, queue_manager, storage, download_strategy_type): +def test_components(url, queue_manager, storage): pass # # component_factory = ComponentFactory(CONFIG) # - # download_strategy = component_factory.get_download_strategy(download_strategy_type) # file_descriptor_manager = component_factory.get_file_descriptor_manager() # # visitor = QueueVisitor( # storage=storage, # callback=component_factory.get_callback(url), # response_strategy=component_factory.get_response_strategy(storage), - # download_strategy=download_strategy, # ) # consumer = Consumer(visitor, queue_manager) #