data AND metadata is being uploaded instead of data only

pyinfra/default_objects.py

@@ -54,7 +54,7 @@ def make_callback(analysis_endpoint):
         try:
             data, metadata = itemgetter("data", "metadata")(body)
             logging.debug(f"Requesting analysis from {endpoint}...")
-            analysis_response_stream = pluck("data", pipeline([data], [metadata]))
+            analysis_response_stream = pipeline([data], [metadata])
             return analysis_response_stream
         except Exception as err:
             logging.warning(f"Exception caught when calling analysis endpoint {endpoint}.")

pyinfra/visitor.py

@@ -7,7 +7,7 @@ from operator import itemgetter
 import random
 from typing import Callable, Generator
 
-from funcy import omit
+from funcy import omit, pluck, first, lmap
 
 from pyinfra.config import CONFIG, parse_disjunction_string
 from pyinfra.exceptions import DataLoadingFailure
@@ -94,7 +94,7 @@ class AggregationStorageStrategy(ResponseStrategy):
 
     def put_object(self, data, metadata):
         object_descriptor = get_response_object_descriptor(metadata)
-        self.storage.put_object(**object_descriptor, data=gzip.compress(string_to_bytes(data)))
+        self.storage.put_object(**object_descriptor, data=gzip.compress(json.dumps(data).encode()))
 
     def merge_queue_items(self):
         merged_buffer_content = self.merger(self.buffer)
@@ -150,6 +150,7 @@ class QueueVisitor:
     def load_and_process(self, body):
         data_from_storage = self.load_data(body)
         result = self.process_data(data_from_storage, body)
+        result = lmap(json.dumps, result)
         result_body = {"data": result, **body}
         return result_body
 

test/integration_tests/serve_test.py

@@ -1,9 +1,12 @@
 import gzip
+import json
 import logging
 
 import pytest
+from funcy import notnone
 
 from pyinfra.default_objects import get_visitor, get_queue_manager, get_storage, get_consumer, get_callback
+from pyinfra.server.packing import bytes_to_string, string_to_bytes
 from pyinfra.visitor import get_object_descriptor
 from test.utils.input import adorn_data_with_storage_info
 
@@ -36,7 +39,13 @@ def test_serving(server_process, input_data_items, bucket_name, endpoint):
         queue_manager.publish_response(req, visitor)
 
     def decode(storage_item):
-        return gzip.decompress(storage_item).decode()
+        repr = gzip.decompress(storage_item).decode().replace(r'\"', "'").replace('"', "").replace("'", '"')
+        try:
+            return json.loads(repr)
+        except json.decoder.JSONDecodeError:
+            return None
 
-    print([*map(decode, storage.get_all_objects(bucket_name))])
-    print([*storage.get_all_object_names(bucket_name)])
+    print()
+    for storage_item in [*filter(notnone, map(decode, storage.get_all_objects(bucket_name)))]:
+        storage_item["data"] = string_to_bytes(storage_item["data"])
+        print(storage_item)