diff --git a/test/exploration_tests/data_json_request_test.py b/test/exploration_tests/data_json_request_test.py index 16dc3fb..b588f91 100644 --- a/test/exploration_tests/data_json_request_test.py +++ b/test/exploration_tests/data_json_request_test.py @@ -1,30 +1,37 @@ -import json -from operator import itemgetter - -import pytest - - -def processor_fn(payload): - data = payload["data"].encode() - metadata = payload["metadata"] - response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))} - return response_payload - - -def test_server_ready_check(client): - response = client.get("/ready") - assert response.status_code == 200 - assert response.json == "OK" - - -@pytest.mark.parametrize("data_type", ["pdf", "bytestring"]) -def test_sending_bytes_through_json(client, data): - payload = {"data": data.decode("latin1"), "metadata": {"A": 1, "B": [2, 3]}} - - response = client.post("/process", json=json.dumps(payload)) - - response_payload = response.json - data_type, metadata_type = itemgetter("data_type", "metadata_type")(response_payload) - - assert data_type == "" - assert metadata_type == "" +# import json +# from operator import itemgetter +# +# import pytest +# import requests +# +# +# def test_server_ready_check(url): +# response = requests.get(f"{url}/ready") +# response.raise_for_status() +# return response.status_code == 200 +# +# +# @pytest.fixture +# def client(client_maker): +# return client_maker(processor_fn) +# +# +# def processor_fn(request): +# payload = json.loads(request.json) +# data = payload["data"].encode() +# metadata = payload["metadata"] +# response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))} +# return response_payload +# +# +# @pytest.mark.parametrize("data_type", ["pdf", "bytestring"]) +# def test_sending_bytes_through_json(url, data): +# payload = {"data": data.decode("latin1"), "metadata": {"A": 1, "B": [2, 3]}} +# +# response = requests.post(f"{url}/process", json=json.dumps(payload)) +# +# response_payload = response.json +# data_type, metadata_type = itemgetter("data_type", "metadata_type")(response_payload) +# +# assert data_type == "" +# assert metadata_type == "" diff --git a/test/exploration_tests/partial_response_test.py b/test/exploration_tests/partial_response_test.py new file mode 100644 index 0000000..0b8c4bd --- /dev/null +++ b/test/exploration_tests/partial_response_test.py @@ -0,0 +1,28 @@ +import json +from itertools import repeat, starmap + +import pytest +import requests + +from test.utils.server import bytes_to_string, string_to_bytes + + +@pytest.mark.parametrize("item_type", ["string"]) +def test_sending_partial_request(url, data_items, metadata): + def pack(metadata: dict, data: bytes): + package = {"data": bytes_to_string(data), "metadata": metadata} + package = "a"# string_to_bytes(json.dumps(package)) + return package + + packages = starmap(pack, zip(repeat(metadata), data_items)) + + def f(): + yield "a" + yield "b" + + requests.post(f"{url}/process", data=packages, stream=True) + + +@pytest.fixture +def metadata(): + return {"idx": [0, 3, 9]} diff --git a/test/fixtures/input.py b/test/fixtures/input.py index 35c4022..3d7a59f 100644 --- a/test/fixtures/input.py +++ b/test/fixtures/input.py @@ -7,3 +7,9 @@ def data(data_type, pdf): return pdf elif data_type == "bytestring": return "content".encode("latin1") + + +@pytest.fixture +def data_items(item_type): + if item_type == "string": + return iter([b"content"] * 5) diff --git a/test/fixtures/server.py b/test/fixtures/server.py index e5ddc13..89d87d9 100644 --- a/test/fixtures/server.py +++ b/test/fixtures/server.py @@ -1,16 +1,82 @@ -import pytest +import socket +from multiprocessing import Process + +import pytest +import requests +from funcy import retry +from waitress import serve -from test.exploration_tests.data_json_request_test import processor_fn from test.server import set_up_processing_server @pytest.fixture -def server(): - server = set_up_processing_server(processor_fn) - server.config.update({"TESTING": True}) - return server +def host(): + return "0.0.0.0" + + +def get_free_port(host): + sock = socket.socket() + sock.bind((host, 0)) + return sock.getsockname()[1] @pytest.fixture -def client(server): - return server.test_client() +def port(host): + return get_free_port(host) + + +@pytest.fixture +def url(host, port): + return f"http://{host}:{port}" + + +@pytest.fixture +def server(processor_fn): + return set_up_processing_server(processor_fn) + + +@pytest.fixture +def processor_fn(item_type, data_items): + if item_type == "string": + return make_string_processor(data_items) + + +def make_string_processor(data_items): + def processor_fn(payload): + print(111111111111111111111111111111111111) + print(payload) + # def parse(package) + # payload = json.load(payload) + # data = string_to_bytes(payload["data"]) + # # response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))} + # return response_payload + + return processor_fn + + +@pytest.fixture +def host_and_port(host, port): + return {"host": host, "port": port} + + +@retry(tries=5, timeout=1) +def server_ready(url): + response = requests.get(f"{url}/ready") + response.raise_for_status() + return response.status_code == 200 + + +@pytest.fixture(autouse=True, scope="function") +def server_process(server, host_and_port, url): + def get_server_process(): + return Process(target=serve, kwargs={"app": server, **host_and_port}) + + server = get_server_process() + server.start() + + if server_ready(url): + yield + + server.kill() + server.join() + server.close() diff --git a/test/server.py b/test/server.py index 33734e6..7cc8f4b 100644 --- a/test/server.py +++ b/test/server.py @@ -1,5 +1,3 @@ -import json - from flask import Flask, jsonify, request @@ -14,8 +12,7 @@ def set_up_processing_server(process_fn): @app.route("/process", methods=["POST"]) def process(): - payload = json.loads(request.json) - response_payload = process_fn(payload) + response_payload = process_fn(request) return jsonify(response_payload) return app diff --git a/test/utils/server.py b/test/utils/server.py new file mode 100644 index 0000000..4e81c1e --- /dev/null +++ b/test/utils/server.py @@ -0,0 +1,9 @@ +import base64 + + +def bytes_to_string(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def string_to_bytes(data: str) -> bytes: + return base64.b64decode(data.encode())