partial response test WIP

test/exploration_tests/data_json_request_test.py

@@ -1,30 +1,37 @@
-import json
-from operator import itemgetter
-
-import pytest
-
-
-def processor_fn(payload):
-    data = payload["data"].encode()
-    metadata = payload["metadata"]
-    response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))}
-    return response_payload
-
-
-def test_server_ready_check(client):
-    response = client.get("/ready")
-    assert response.status_code == 200
-    assert response.json == "OK"
-
-
-@pytest.mark.parametrize("data_type", ["pdf", "bytestring"])
-def test_sending_bytes_through_json(client, data):
-    payload = {"data": data.decode("latin1"), "metadata": {"A": 1, "B": [2, 3]}}
-
-    response = client.post("/process", json=json.dumps(payload))
-
-    response_payload = response.json
-    data_type, metadata_type = itemgetter("data_type", "metadata_type")(response_payload)
-
-    assert data_type == "<class 'bytes'>"
-    assert metadata_type == "<class 'dict'>"
+# import json
+# from operator import itemgetter
+#
+# import pytest
+# import requests
+#
+#
+# def test_server_ready_check(url):
+#     response = requests.get(f"{url}/ready")
+#     response.raise_for_status()
+#     return response.status_code == 200
+#
+#
+# @pytest.fixture
+# def client(client_maker):
+#     return client_maker(processor_fn)
+#
+#
+# def processor_fn(request):
+#     payload = json.loads(request.json)
+#     data = payload["data"].encode()
+#     metadata = payload["metadata"]
+#     response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))}
+#     return response_payload
+#
+#
+# @pytest.mark.parametrize("data_type", ["pdf", "bytestring"])
+# def test_sending_bytes_through_json(url, data):
+#     payload = {"data": data.decode("latin1"), "metadata": {"A": 1, "B": [2, 3]}}
+#
+#     response = requests.post(f"{url}/process", json=json.dumps(payload))
+#
+#     response_payload = response.json
+#     data_type, metadata_type = itemgetter("data_type", "metadata_type")(response_payload)
+#
+#     assert data_type == "<class 'bytes'>"
+#     assert metadata_type == "<class 'dict'>"

test/exploration_tests/partial_response_test.py

@@ -0,0 +1,28 @@
+import json
+from itertools import repeat, starmap
+
+import pytest
+import requests
+
+from test.utils.server import bytes_to_string, string_to_bytes
+
+
+@pytest.mark.parametrize("item_type", ["string"])
+def test_sending_partial_request(url, data_items, metadata):
+    def pack(metadata: dict, data: bytes):
+        package = {"data": bytes_to_string(data), "metadata": metadata}
+        package = "a"# string_to_bytes(json.dumps(package))
+        return package
+
+    packages = starmap(pack, zip(repeat(metadata), data_items))
+
+    def f():
+        yield "a"
+        yield "b"
+
+    requests.post(f"{url}/process", data=packages, stream=True)
+
+
+@pytest.fixture
+def metadata():
+    return {"idx": [0, 3, 9]}

test/fixtures/input.py

@@ -7,3 +7,9 @@ def data(data_type, pdf):
         return pdf
     elif data_type == "bytestring":
         return "content".encode("latin1")
+
+
+@pytest.fixture
+def data_items(item_type):
+    if item_type == "string":
+        return iter([b"content"] * 5)

test/fixtures/server.py

@@ -1,16 +1,82 @@
-import pytest
+import socket
+from multiprocessing import Process
+
+import pytest
+import requests
+from funcy import retry
+from waitress import serve
 
-from test.exploration_tests.data_json_request_test import processor_fn
 from test.server import set_up_processing_server
 
 
 @pytest.fixture
-def server():
-    server = set_up_processing_server(processor_fn)
-    server.config.update({"TESTING": True})
-    return server
+def host():
+    return "0.0.0.0"
+
+
+def get_free_port(host):
+    sock = socket.socket()
+    sock.bind((host, 0))
+    return sock.getsockname()[1]
 
 
 @pytest.fixture
-def client(server):
-    return server.test_client()
+def port(host):
+    return get_free_port(host)
+
+
+@pytest.fixture
+def url(host, port):
+    return f"http://{host}:{port}"
+
+
+@pytest.fixture
+def server(processor_fn):
+    return set_up_processing_server(processor_fn)
+
+
+@pytest.fixture
+def processor_fn(item_type, data_items):
+    if item_type == "string":
+        return make_string_processor(data_items)
+
+
+def make_string_processor(data_items):
+    def processor_fn(payload):
+        print(111111111111111111111111111111111111)
+        print(payload)
+        # def parse(package)
+        # payload = json.load(payload)
+        # data = string_to_bytes(payload["data"])
+        # # response_payload = {"metadata_type": str(type(metadata)), "data_type": str(type(data))}
+        # return response_payload
+
+    return processor_fn
+
+
+@pytest.fixture
+def host_and_port(host, port):
+    return {"host": host, "port": port}
+
+
+@retry(tries=5, timeout=1)
+def server_ready(url):
+    response = requests.get(f"{url}/ready")
+    response.raise_for_status()
+    return response.status_code == 200
+
+
+@pytest.fixture(autouse=True, scope="function")
+def server_process(server, host_and_port, url):
+    def get_server_process():
+        return Process(target=serve, kwargs={"app": server, **host_and_port})
+
+    server = get_server_process()
+    server.start()
+
+    if server_ready(url):
+        yield
+
+    server.kill()
+    server.join()
+    server.close()

test/server.py

@@ -1,5 +1,3 @@
-import json
-
 from flask import Flask, jsonify, request
 
 
@@ -14,8 +12,7 @@ def set_up_processing_server(process_fn):
 
     @app.route("/process", methods=["POST"])
     def process():
-        payload = json.loads(request.json)
-        response_payload = process_fn(payload)
+        response_payload = process_fn(request)
         return jsonify(response_payload)
 
     return app

test/utils/server.py

@@ -0,0 +1,9 @@
+import base64
+
+
+def bytes_to_string(data: bytes) -> str:
+    return base64.b64encode(data).decode()
+
+
+def string_to_bytes(data: str) -> bytes:
+    return base64.b64decode(data.encode())