1363030b95
Merge in RR/pyinfra from kbudisantoso/configyaml-1650538128334 to master
Squashed commit of the following:
commit 6103b7720315aaef3d98aea8f3c817477bbf500b
Merge: 69ac65a 3b91185
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Thu Apr 21 14:45:10 2022 +0200
Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334
commit 69ac65ae1bd4095c797112c6f9530f0b1705277e
Merge: 9a1cd07 a00ceae
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Thu Apr 21 14:37:34 2022 +0200
Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334
commit 9a1cd07c09e5ee2618f2c1a3c27b69c67b1eaeb0
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Thu Apr 21 14:35:49 2022 +0200
test done
commit e7127e8af937fe067f1f92eb688187ebbe609478
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Thu Apr 21 14:32:25 2022 +0200
test
commit 262957e33d19dbafb3f10b5a32c438460b966a88
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Thu Apr 21 14:16:33 2022 +0200
add parser for env var storage_buckets/containers
commit 3535002b4aac9f297bdbe112b04f537cef25f5c2
Author: Kresnadi Budisantoso <kresnadi.budisantoso@iqser.com>
Date: Thu Apr 21 12:48:52 2022 +0200
config.yaml online editiert mit Bitbucket
73 lines
2.3 KiB
Python
73 lines
2.3 KiB
Python
import argparse
|
|
import gzip
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from tqdm import tqdm
|
|
|
|
from pyinfra.config import CONFIG, parse_disjunction_string
|
|
from pyinfra.storage.storages import get_s3_storage
|
|
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser()
|
|
|
|
subparsers = parser.add_subparsers(help="sub-command help", dest="command")
|
|
|
|
parser_add = subparsers.add_parser("add", help="Add file(s) to the MinIO store")
|
|
parser_add.add_argument("dossier_id")
|
|
add_group = parser_add.add_mutually_exclusive_group(required=True)
|
|
add_group.add_argument("--file", "-f")
|
|
add_group.add_argument("--directory", "-d")
|
|
|
|
subparsers.add_parser("purge", help="Delete all files and buckets in the MinIO store")
|
|
|
|
args = parser.parse_args()
|
|
return args
|
|
|
|
|
|
def combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, extension):
|
|
return f"{dossier_id}/{file_id}{extension}"
|
|
|
|
|
|
def upload_compressed_response(storage, bucket_name, dossier_id, file_id, result) -> None:
|
|
data = gzip.compress(result.encode())
|
|
path_gz = combine_dossier_id_and_file_id_and_extension(dossier_id, file_id, CONFIG.service.response.extension)
|
|
storage.put_object(bucket_name, path_gz, data)
|
|
|
|
|
|
def add_file_compressed(storage, bucket_name, dossier_id, path) -> None:
|
|
if Path(path).suffix == ".pdf":
|
|
suffix_gz = ".ORIGIN.pdf.gz"
|
|
if Path(path).suffix == ".json":
|
|
suffix_gz = ".TEXT.json.gz"
|
|
path_gz = combine_dossier_id_and_file_id_and_extension(dossier_id, Path(path).stem, suffix_gz)
|
|
|
|
with open(path, "rb") as f:
|
|
data = gzip.compress(f.read())
|
|
storage.put_object(bucket_name, path_gz, data)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
storage = get_s3_storage()
|
|
bucket_name = parse_disjunction_string(CONFIG.storage.bucket)
|
|
|
|
if not storage.has_bucket(bucket_name):
|
|
storage.make_bucket(bucket_name)
|
|
|
|
args = parse_args()
|
|
|
|
if args.command == "add":
|
|
|
|
if args.file:
|
|
add_file_compressed(storage, bucket_name, args.dossier_id, args.file)
|
|
|
|
elif args.directory:
|
|
for fname in tqdm([*os.listdir(args.directory)], desc="Adding files"):
|
|
path = Path(args.directory) / fname
|
|
add_file_compressed(storage, bucket_name, args.dossier_id, path)
|
|
|
|
elif args.command == "purge":
|
|
storage.clear_bucket(bucket_name)
|