diff --git a/config.yaml b/config.yaml index 6e6aa6c..2eb691d 100755 --- a/config.yaml +++ b/config.yaml @@ -5,9 +5,6 @@ service: # TODO: will become obsolete by below changes response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for response files to upload to storage - # In case a service is not supposed to place response files under the root folder `dossierId/fileId` and requests do - # not specify an operation, this parameter can specify an output folder - response_folder: $RESPONSE_FOLDER|null # FIXME: covered by operation mapping? # Specifies, how to handle the `page` key of a request. "multi" will download all pages matching the list of pages # specified in the request download_strategy: $DOWNLOAD_STRATEGY|multi diff --git a/pyinfra/file_descriptor_manager.py b/pyinfra/file_descriptor_manager.py index 7108c06..ab2f18e 100644 --- a/pyinfra/file_descriptor_manager.py +++ b/pyinfra/file_descriptor_manager.py @@ -1,11 +1,8 @@ -import json import os from _operator import itemgetter from funcy import project -from pyinfra.config import CONFIG - class FileDescriptorManager: def __init__(self, bucket_name, operation2file_patterns: dict = None): @@ -24,18 +21,12 @@ class FileDescriptorManager: def get_object_name(self, queue_item_body: dict, end): - # TODO: refactor + file_descriptor = self.build_file_descriptor(queue_item_body, end=end) + file_descriptor["pages"] = [queue_item_body.get("id", 0)] - if end == "output": - return self.get_response_object_name(queue_item_body) - else: + object_name = self.__build_matcher(file_descriptor) - file_descriptor = self.build_file_descriptor(queue_item_body, end=end) - file_descriptor["pages"] = [queue_item_body.get("id", 0)] - - object_name = self.__build_matcher(file_descriptor) - - return object_name + return object_name @staticmethod def __build_matcher(file_descriptor): @@ -74,7 +65,7 @@ class FileDescriptorManager: return self.build_matcher(queue_item_body, end="input") def build_output_matcher(self, queue_item_body): - return self.get_response_object_name(queue_item_body) + return self.build_matcher(queue_item_body, end="output") def build_matcher(self, queue_item_body, end): file_descriptor = self.build_file_descriptor(queue_item_body, end=end) @@ -96,16 +87,3 @@ class FileDescriptorManager: def build_storage_upload_info(analysis_payload, request_metadata): storage_upload_info = {**request_metadata, "id": analysis_payload["metadata"].get("id", 0)} return storage_upload_info - - @staticmethod - def get_response_object_name(body): - # TODO: refactor - - if "id" not in body: - body["id"] = 0 - - dossier_id, file_id, idnt = itemgetter("dossierId", "fileId", "id")(body) - - object_name = f"{dossier_id}/{file_id}/id:{idnt}.{CONFIG.service.response_file_extension}" - - return object_name diff --git a/pyinfra/visitor/strategies/response/aggregation.py b/pyinfra/visitor/strategies/response/aggregation.py index 098f6a0..32713cb 100644 --- a/pyinfra/visitor/strategies/response/aggregation.py +++ b/pyinfra/visitor/strategies/response/aggregation.py @@ -38,11 +38,6 @@ class AggregationStorageStrategy(ResponseStrategy): """analysis_payload : {data: ..., metadata: ...}""" storage_upload_info = self.file_descriptor_manager.build_storage_upload_info(analysis_payload, request_metadata) object_descriptor = self.file_descriptor_manager.get_output_object_descriptor(storage_upload_info) - print(111111111) - print(json.dumps(storage_upload_info, indent=2)) - print(222222222) - print(json.dumps(object_descriptor, indent=2)) - print() self.add_analysis_payload_to_buffer(analysis_payload) diff --git a/test/config.yaml b/test/config.yaml index 228f397..ded376f 100644 --- a/test/config.yaml +++ b/test/config.yaml @@ -13,7 +13,7 @@ service: subdir: "" extension: IN.gz output: - subdir: "" + subdir: op_outp_files extension: OUT.gz default: input: diff --git a/test/integration_tests/serve_test.py b/test/integration_tests/serve_test.py index 1741033..99e95ec 100644 --- a/test/integration_tests/serve_test.py +++ b/test/integration_tests/serve_test.py @@ -221,8 +221,6 @@ def real_components(url, download_strategy_type): storage = component_factory.get_storage() file_descriptor_manager = component_factory.get_file_descriptor_manager() - print(download_strategy_type, component_factory.get_download_strategy()) - return storage, queue_manager, consumer, file_descriptor_manager