Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

test config options for logging and docker; changed object name construction

Browse Source
This commit is contained in:
Matthias Bisping 2022-06-15 09:59:47 +02:00
parent 147416bfad
commit 24be8d3f13
4 changed files with 28 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config.yaml
View File

@ -1,6 +1,7 @@
service: service:
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger
response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for files uploaded to storage target_file_extension: $TARGET_FILE_EXTENSION|"pdf.gz" # Extension for files to download from storage and process
response_file_extension: $RESPONSE_FILE_EXTENSION|"json.gz" # Extension for response files to upload to storage
# In case a service is not supposed to place response files under the root folder `dossierId/fileId` and requests do # In case a service is not supposed to place response files under the root folder `dossierId/fileId` and requests do
# not specify an operation, this parameter can specify an output folder # not specify an operation, this parameter can specify an output folder
response_folder: $RESPONSE_FOLDER|null response_folder: $RESPONSE_FOLDER|null

22
pyinfra/visitor.py
View File

@ -8,7 +8,7 @@ from copy import deepcopy
from operator import itemgetter from operator import itemgetter
from typing import Callable, Dict, Union from typing import Callable, Dict, Union
from funcy import omit, filter, lmap, lflatten from funcy import omit, filter, lflatten
from more_itertools import peekable from more_itertools import peekable
from pyinfra.config import CONFIG, parse_disjunction_string from pyinfra.config import CONFIG, parse_disjunction_string
@ -31,21 +31,19 @@ def unique_hash(pages):
return hsh return hsh
def get_object_name(body): def get_object_name(body: dict):
def get_key(key):
return key if key in body else False
body = deepcopy(body) body = deepcopy(body)
if "pages" not in body: folder = f"/{folder}/" if (folder := get_key("pages") or get_key("images")) else ""
body["pages"] = [] idnt = f"id:{idnt}" if (idnt := body.get("id", "0" if folder else False)) else ""
if "id" not in body: dossier_id, file_id = itemgetter("dossierId", "fileId")(body)
body["id"] = 0
dossier_id, file_id, pages, idnt, target_file_extension = itemgetter( object_name = f"{dossier_id}/{file_id}{folder}{idnt}.{CONFIG.service.target_file_extension}"
"dossierId", "fileId", "pages", "id", "targetFileExtension"
)(body)
object_name = f"{dossier_id}/{file_id}/{unique_hash(pages)}-id:{idnt}.{CONFIG.service.response_file_extension}"
return object_name return object_name
@ -339,7 +337,7 @@ class MultiDownloadStrategy(DownloadStrategy):
def download(self, storage: Storage, queue_item_body): def download(self, storage: Storage, queue_item_body):
pages = "|".join(map(str, queue_item_body["pages"])) pages = "|".join(map(str, queue_item_body["pages"]))
matches_page = r".*-id:(" + pages + r").*" matches_page = r".*id:(" + pages + r").*"
object_names = storage.get_all_object_names(self.bucket_name) object_names = storage.get_all_object_names(self.bucket_name)
object_names = filter(matches_page, object_names) object_names = filter(matches_page, object_names)

5
test/config.yaml
View File

@ -22,4 +22,7 @@ webserver:
mode: $SERVER_MODE|production # webserver mode: {development, production} mode: $SERVER_MODE|production # webserver mode: {development, production}
mock_analysis_endpoint: "http://127.0.0.1:5000" mock_analysis_endpoint: "http://127.0.0.1:5000"
use_docker_fixture: False
logging: False

21
test/conftest.py
View File

@ -40,9 +40,10 @@ logging.getLogger("PIL.PngImagePlugin").setLevel(level=logging.CRITICAL + 1)
logging.getLogger("waitress").setLevel(level=logging.CRITICAL + 1) logging.getLogger("waitress").setLevel(level=logging.CRITICAL + 1)
@pytest.fixture(autouse=False) @pytest.fixture(autouse=True)
def mute_logger(): def mute_logger():
logger.setLevel(logging.CRITICAL + 1) if not CONFIG.logging:
logger.setLevel(logging.CRITICAL + 1)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@ -99,13 +100,15 @@ def s3_backend(request):
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
def docker_compose(sleep_seconds=30): def docker_compose(sleep_seconds=30):
logger.info(f"Starting docker containers with {COMPOSE_PATH}/docker-compose.yml...") if CONFIG.use_docker_fixture:
compose = DockerCompose(COMPOSE_PATH, compose_file_name="docker-compose.yml") logger.info(f"Starting docker containers with {COMPOSE_PATH}/docker-compose.yml...")
compose.start() compose = DockerCompose(COMPOSE_PATH, compose_file_name="docker-compose.yml")
logger.info(f"Sleeping for {sleep_seconds} seconds to wait for containers to finish startup... ") compose.start()
time.sleep(sleep_seconds) logger.info(f"Sleeping for {sleep_seconds} seconds to wait for containers to finish startup... ")
yield compose time.sleep(sleep_seconds)
compose.stop() yield compose
compose.stop()
yield None
def get_pika_connection_params(): def get_pika_connection_params():