Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

add AES/GCM cipher functions

Browse Source 
- decrypt x-tenant storage connection strings
This commit is contained in:
Julius Unverfehrt 2023-03-21 15:35:57 +01:00
parent ff6f437e84
commit 52c047c47b
5 changed files with 129 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
poetry.lock generated
View File

@ -1360,6 +1360,49 @@ files = [
{file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
]
[[package]]
name = "pycryptodome"
version = "3.17"
description = "Cryptographic library for Python"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
files = [
{file = "pycryptodome-3.17-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:2c5631204ebcc7ae33d11c43037b2dafe25e2ab9c1de6448eb6502ac69c19a56"},
{file = "pycryptodome-3.17-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:04779cc588ad8f13c80a060b0b1c9d1c203d051d8a43879117fe6b8aaf1cd3fa"},
{file = "pycryptodome-3.17-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:f812d58c5af06d939b2baccdda614a3ffd80531a26e5faca2c9f8b1770b2b7af"},
{file = "pycryptodome-3.17-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:9453b4e21e752df8737fdffac619e93c9f0ec55ead9a45df782055eb95ef37d9"},
{file = "pycryptodome-3.17-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:121d61663267f73692e8bde5ec0d23c9146465a0d75cad75c34f75c752527b01"},
{file = "pycryptodome-3.17-cp27-cp27m-win32.whl", hash = "sha256:ba2d4fcb844c6ba5df4bbfee9352ad5352c5ae939ac450e06cdceff653280450"},
{file = "pycryptodome-3.17-cp27-cp27m-win_amd64.whl", hash = "sha256:87e2ca3aa557781447428c4b6c8c937f10ff215202ab40ece5c13a82555c10d6"},
{file = "pycryptodome-3.17-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:f44c0d28716d950135ff21505f2c764498eda9d8806b7c78764165848aa419bc"},
{file = "pycryptodome-3.17-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5a790bc045003d89d42e3b9cb3cc938c8561a57a88aaa5691512e8540d1ae79c"},
{file = "pycryptodome-3.17-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:d086d46774e27b280e4cece8ab3d87299cf0d39063f00f1e9290d096adc5662a"},
{file = "pycryptodome-3.17-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:5587803d5b66dfd99e7caa31ed91fba0fdee3661c5d93684028ad6653fce725f"},
{file = "pycryptodome-3.17-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:e7debd9c439e7b84f53be3cf4ba8b75b3d0b6e6015212355d6daf44ac672e210"},
{file = "pycryptodome-3.17-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ca1ceb6303be1282148f04ac21cebeebdb4152590842159877778f9cf1634f09"},
{file = "pycryptodome-3.17-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:dc22cc00f804485a3c2a7e2010d9f14a705555f67020eb083e833cabd5bd82e4"},
{file = "pycryptodome-3.17-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80ea8333b6a5f2d9e856ff2293dba2e3e661197f90bf0f4d5a82a0a6bc83a626"},
{file = "pycryptodome-3.17-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c133f6721fba313722a018392a91e3c69d3706ae723484841752559e71d69dc6"},
{file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:333306eaea01fde50a73c4619e25631e56c4c61bd0fb0a2346479e67e3d3a820"},
{file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:1a30f51b990994491cec2d7d237924e5b6bd0d445da9337d77de384ad7f254f9"},
{file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:909e36a43fe4a8a3163e9c7fc103867825d14a2ecb852a63d3905250b308a4e5"},
{file = "pycryptodome-3.17-cp35-abi3-win32.whl", hash = "sha256:a3228728a3808bc9f18c1797ec1179a0efb5068c817b2ffcf6bcd012494dffb2"},
{file = "pycryptodome-3.17-cp35-abi3-win_amd64.whl", hash = "sha256:9ec565e89a6b400eca814f28d78a9ef3f15aea1df74d95b28b7720739b28f37f"},
{file = "pycryptodome-3.17-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:e1819b67bcf6ca48341e9b03c2e45b1c891fa8eb1a8458482d14c2805c9616f2"},
{file = "pycryptodome-3.17-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:f8e550caf52472ae9126953415e4fc554ab53049a5691c45b8816895c632e4d7"},
{file = "pycryptodome-3.17-pp27-pypy_73-win32.whl", hash = "sha256:afbcdb0eda20a0e1d44e3a1ad6d4ec3c959210f4b48cabc0e387a282f4c7deb8"},
{file = "pycryptodome-3.17-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a74f45aee8c5cc4d533e585e0e596e9f78521e1543a302870a27b0ae2106381e"},
{file = "pycryptodome-3.17-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38bbd6717eac084408b4094174c0805bdbaba1f57fc250fd0309ae5ec9ed7e09"},
{file = "pycryptodome-3.17-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f68d6c8ea2974a571cacb7014dbaada21063a0375318d88ac1f9300bc81e93c3"},
{file = "pycryptodome-3.17-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8198f2b04c39d817b206ebe0db25a6653bb5f463c2319d6f6d9a80d012ac1e37"},
{file = "pycryptodome-3.17-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3a232474cd89d3f51e4295abe248a8b95d0332d153bf46444e415409070aae1e"},
{file = "pycryptodome-3.17-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4992ec965606054e8326e83db1c8654f0549cdb26fce1898dc1a20bc7684ec1c"},
{file = "pycryptodome-3.17-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53068e33c74f3b93a8158dacaa5d0f82d254a81b1002e0cd342be89fcb3433eb"},
{file = "pycryptodome-3.17-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:74794a2e2896cd0cf56fdc9db61ef755fa812b4a4900fa46c49045663a92b8d0"},
{file = "pycryptodome-3.17.tar.gz", hash = "sha256:bce2e2d8e82fcf972005652371a3e8731956a0c1fbb719cc897943b3695ad91b"},
]
[[package]]
name = "pygments"
version = "2.14.0"
@ -2041,4 +2084,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
[metadata]
lock-version = "2.0"
python-versions = "~3.8"
content-hash = "d5017298c5fddc7d919a319fec7e050e8f7603c0c11c08d5c6c3115e751083e2"
content-hash = "43b21cbd83ef95d9d28a72856f9b2d78ad4f0c8c740af58bf8b40d1c61dba50d"

12
pyinfra/payload_processing/processor.py
View File

@ -38,16 +38,16 @@ class PayloadProcessor:
payload_parser: Parser that translates the queue message payload to the required QueueMessagePayload object
payload_formatter: Formatter for the storage upload result and the queue message response body
data_processor: The analysis function to be called with the downloaded file
NOTE: the result of the analysis function has to be Sized, e.g. a dict or a list to be able to upload it
and to be able to monitor the processing time.
NOTE: The result of the analysis function has to be an instance of `Sized`, e.g. a dict or a list to be
able to upload it and to be able to monitor the processing time.
"""
self.parse_payload = payload_parser
self.format_result_for_storage = payload_formatter.format_service_processing_result_for_storage
self.format_to_queue_message_response_body = payload_formatter.format_to_queue_message_response_body
self.process_data = data_processor
self.partial_download_fn = partial(make_downloader, storage, bucket)
self.partial_upload_fn = partial(make_uploader, storage, bucket)
self.make_downloader = partial(make_downloader, storage, bucket)
self.make_uploader = partial(make_uploader, storage, bucket)
def __call__(self, queue_message_payload: dict) -> dict:
"""Processes a queue message payload.
@ -71,8 +71,8 @@ class PayloadProcessor:
payload = self.parse_payload(queue_message_payload)
logger.info(f"Processing {asdict(payload)} ...")
download_file_to_process = self.partial_download_fn(payload.target_file_type, payload.target_compression_type)
upload_processing_result = self.partial_upload_fn(payload.response_file_type, payload.response_compression_type)
download_file_to_process = self.make_downloader(payload.target_file_type, payload.target_compression_type)
upload_processing_result = self.make_uploader(payload.response_file_type, payload.response_compression_type)
format_result_for_storage = partial(self.format_result_for_storage, payload)
processing_pipeline = compose(format_result_for_storage, self.process_data, download_file_to_process)

49
pyinfra/utils/cipher.py Normal file
View File

@ -0,0 +1,49 @@
import base64
import os
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
def build_aes_gcm_cipher(public_key, iv=None):
encoded_key = public_key.encode("utf-8")
kdf = PBKDF2HMAC(
algorithm=hashes.SHA1(),
length=16,
salt=iv,
iterations=65536,
)
private_key = kdf.derive(encoded_key)
return AESGCM(private_key)
def encrypt(public_key: str, plaintext: str, iv: int = None) -> str:
"""Encrypt a text with AES/GCS using a public key.
The byte-converted ciphertext consists of an unsigned 32-bit integer big-endian byteorder header i.e. the first 4
bytes, specifying the length of the following initialization vector (iv). The rest of the text contains the
encrypted message.
"""
iv = iv or os.urandom(12)
plaintext_bytes = plaintext.encode("utf-8")
cipher = build_aes_gcm_cipher(public_key, iv)
header = len(iv).to_bytes(length=4, byteorder="big")
encrypted = header + iv + cipher.encrypt(nonce=iv, data=plaintext_bytes, associated_data=None)
return base64.b64encode(encrypted).decode("utf-8")
def decrypt(public_key: str, ciphertext: str) -> str:
"""Decrypt an AES/GCS encrypted text with a public key.
The byte-converted ciphertext consists of an unsigned 32-bit integer big-endian byteorder header i.e. the first 4
bytes, specifying the length of the following initialization vector (iv). The rest of the text contains the
encrypted message.
"""
ciphertext_bytes = base64.b64decode(ciphertext)
header, rest = ciphertext_bytes[:4], ciphertext_bytes[4:]
iv_length = int.from_bytes(header, "big")
iv, ciphertext_bytes = rest[:iv_length], rest[iv_length:]
cipher = build_aes_gcm_cipher(public_key, iv)
decrypted_text = cipher.decrypt(nonce=iv, data=ciphertext_bytes, associated_data=None)
return decrypted_text.decode("utf-8")

1
pyproject.toml
View File

@ -17,6 +17,7 @@ testcontainers = "3.4.2"
docker-compose = "1.29.2"
funcy = "1.17"
prometheus-client = "^0.16.0"
pycryptodome = "^3.17"
[tool.poetry.group.dev.dependencies]
pytest = "^7.1.3"

29
tests/cipher_test.py Normal file
View File

@ -0,0 +1,29 @@
import pytest
from pyinfra.utils.cipher import decrypt, encrypt
@pytest.fixture
def ciphertext():
return "AAAADBRzag4/aAE2+rSekyI5phVZ1e0wwSaRkGQTLftPyVvq8vLYZzwxW48Wozc3/w=="
@pytest.fixture
def plaintext():
return "connectzionString"
@pytest.fixture
def public_key():
return "redaction"
class TestDecryption:
def test_decrypt_ciphertext(self, public_key, ciphertext, plaintext):
result = decrypt(public_key, ciphertext)
assert result == plaintext
def test_encrypt_plaintext(self, public_key, plaintext):
ciphertext = encrypt(public_key, plaintext)
result = decrypt(public_key, ciphertext)
assert plaintext == result