Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

refactoring; added operations to test config + overwriting main config in components fixture

Browse Source
This commit is contained in:
Matthias Bisping 2022-06-21 16:59:43 +02:00
parent 21daab2404
commit 3f03e25bd3
4 changed files with 24 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
pyinfra/visitor/strategies/download/multi.py
View File

@ -24,19 +24,6 @@ class MultiDownloadStrategy(DownloadStrategy):
def __call__(self, storage, queue_item_body): def __call__(self, storage, queue_item_body):
return self.download(storage, queue_item_body) return self.download(storage, queue_item_body)
def get_page_matcher(self, queue_item_body):
pages = queue_item_body.get("pages", None)
if "pages" in queue_item_body:
pages = "|".join(map(str, pages))
page_matcher = "pages/id:(" + pages + ")."
else:
page_matcher = ""
return page_matcher
def get_names_of_objects_by_pages(self, storage, file_pattern: str): def get_names_of_objects_by_pages(self, storage, file_pattern: str):
matches_pattern = flift(file_pattern) matches_pattern = flift(file_pattern)
page_object_names = compose(matches_pattern, storage.get_all_object_names)(self.bucket_name) page_object_names = compose(matches_pattern, storage.get_all_object_names)(self.bucket_name)
@ -86,3 +73,16 @@ class MultiDownloadStrategy(DownloadStrategy):
file_pattern = file_pattern.replace("<id>", self.get_page_matcher(queue_item_body)) file_pattern = file_pattern.replace("<id>", self.get_page_matcher(queue_item_body))
return file_pattern return file_pattern
def get_page_matcher(self, queue_item_body):
pages = queue_item_body.get("pages", None)
if "pages" in queue_item_body:
pages = "|".join(map(str, pages))
page_matcher = "pages/id:(" + pages + ")."
else:
page_matcher = ""
return page_matcher

7
test/config.yaml
View File

@ -1,3 +1,9 @@
service:
response_formatter: identity
operations:
default:
input_file_pattern: <dossier_id>/<file_id>/<id>ORIGIN.pdf.gz
storage: storage:
minio: minio:
endpoint: "http://127.0.0.1:9000" endpoint: "http://127.0.0.1:9000"
@ -21,7 +27,6 @@ webserver:
port: $SERVER_PORT|5000 # webserver port port: $SERVER_PORT|5000 # webserver port
mode: $SERVER_MODE|production # webserver mode: {development, production} mode: $SERVER_MODE|production # webserver mode: {development, production}
mock_analysis_endpoint: "http://127.0.0.1:5000" mock_analysis_endpoint: "http://127.0.0.1:5000"
use_docker_fixture: 0 use_docker_fixture: 0

1
test/conftest.py
View File

@ -4,7 +4,6 @@ from pyinfra.config import CONFIG as MAIN_CONFIG
MAIN_CONFIG["retry"]["delay"] = 0.1 MAIN_CONFIG["retry"]["delay"] = 0.1
MAIN_CONFIG["retry"]["jitter"] = (0.1, 0.2) MAIN_CONFIG["retry"]["jitter"] = (0.1, 0.2)
MAIN_CONFIG["service"]["response_formatter"] = "identity"
import logging import logging
import time import time

5
test/integration_tests/serve_test.py
View File

@ -208,6 +208,11 @@ def components_type(request):
@pytest.fixture @pytest.fixture
def real_components(url, download_strategy): def real_components(url, download_strategy):
from test.config import CONFIG as TEST_CONFIG
CONFIG["service"]["operations"] = TEST_CONFIG.service.operations
CONFIG["service"]["response_formatter"] = TEST_CONFIG.service.response_formatter
component_factory = get_component_factory(CONFIG) component_factory = get_component_factory(CONFIG)
callback = component_factory.get_callback(url) callback = component_factory.get_callback(url)