dca206cda5
Merge in RR/pyinfra from s3_refactoring to master
Squashed commit of the following:
commit d9224b879d47c6f1574d78b7239a209f800972dc
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 12:52:57 2022 +0100
fixed brokem reference
commit 4189ebb0481c85b5984adb8c6008bbc9bae6d71b
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 12:51:11 2022 +0100
renaming
commit 945f2397c5301409be19229e11bb50a733f6e2ff
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 12:48:13 2022 +0100
renaming
commit 3d86500b4b703a56cb32d998c1c12e8fd06e3b2c
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 12:45:32 2022 +0100
renaming
commit 4b26acca978ad49bb0b6e785addcb601d90214db
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 12:39:08 2022 +0100
renaming
commit 81be5e55b6d70ccb9f8b07b5860f89da7231122f
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 11:44:46 2022 +0100
renaming
commit 23b35b1f91def3a6185315772c1249826637e627
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Fri Feb 25 11:31:15 2022 +0100
added teardown step of clearing test bucke to storage tests; added tests for nested paths
commit 6e26c1a6962bfe867827f80a38c091f63fa395f0
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Fri Feb 25 10:56:32 2022 +0100
black & removed käsebrot
commit 519c88b39933fa699e6eff5d29bfea7ed1ca8719
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Fri Feb 25 10:55:18 2022 +0100
data loading/publishing adjusted to new storage logic
commit c530b5929a6f892663009eef4b92f04b4f7f533c
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 17:24:47 2022 +0100
removing old storage logic WIP
commit 880e22832cd203dc0d0471729996cbd0b5f455f1
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 17:12:44 2022 +0100
manage minio script refactor
commit 01149770750bdf88094376b1f731ace343f32d76
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 16:32:23 2022 +0100
renaming
commit 7d6dbabf30d928cb699564e2e7a668253ec6990b
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 16:26:10 2022 +0100
added list bucket files & refactor serve.py to take new storage logic
commit b20d6fd26b4523538fd02acc5889b932de2e3f03
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Thu Feb 24 15:42:23 2022 +0100
added compose file referecen to pytest.ini; added coveragerc
commit ff1c7e319bd229bc5dd905ddbc04c211b78f88cd
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Thu Feb 24 15:41:13 2022 +0100
added pytest.ini
commit d32828ad917a20cf16785a31008d90b4f2ed4442
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 15:37:30 2022 +0100
black
commit 36931850c3d720b76d39a9f503abeaa326132531
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Thu Feb 24 15:35:55 2022 +0100
get storage test added
commit 0eb7aaf795217bb34f8f73b5e1e90576b55e9374
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Thu Feb 24 15:08:29 2022 +0100
added fixture magic for testing differen S3 backends
commit f3f1c42e574365702be632a416b07f025746e989
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Thu Feb 24 14:47:03 2022 +0100
defined test sections in config for enpoints of storages
commit 16e34052da2cc8c61473d21c1929023445e619d6
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date: Thu Feb 24 14:11:53 2022 +0100
refactoring; implemented S3 handle
... and 18 more commits
60 lines
2.6 KiB
YAML
Executable File
60 lines
2.6 KiB
YAML
Executable File
rabbitmq:
|
|
host: $RABBITMQ_HOST|localhost # RabbitMQ host address
|
|
port: $RABBITMQ_PORT|5672 # RabbitMQ host port
|
|
user: $RABBITMQ_USERNAME|user # RabbitMQ username
|
|
password: $RABBITMQ_PASSWORD|bitnami # RabbitMQ password
|
|
heartbeat: $RABBITMQ_HEARTBEAT|7200 # Controls AMQP heartbeat timeout in seconds
|
|
|
|
queues:
|
|
input: image_request_queue # Requests to service
|
|
output: image_response_queue # Responses by service
|
|
dead_letter: image_dead_letter_queue # Messages that failed to process
|
|
|
|
prefetch_count: 1
|
|
|
|
retry: # Controls retry behaviour for messages the processing of which failed
|
|
enabled: $RETRY|False # Toggles retry behaviour
|
|
max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue
|
|
|
|
s3:
|
|
endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000" # MinIO endpoint
|
|
access_key: $STORAGE_KEY|root # MinIO user name
|
|
secret_key: $STORAGE_SECRET|password # MinIO user password
|
|
bucket: $STORAGE_BUCKET_NAME|redaction # MinIO bucket
|
|
|
|
azure_blob_storage:
|
|
connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
|
|
container: $STORAGE_AZURECONTAINERNAME|"pyinfra-test-storage"
|
|
|
|
service:
|
|
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger
|
|
name: $SERVICE_NAME|pyinfra-service-v1 # Name of the service in the kubernetes cluster
|
|
storage_backend: $STORAGE_BACKEND|s3 # The storage to pull files to be processed from
|
|
analysis_endpoint: $ANALYSIS_ENDPOINT|"http://127.0.0.1:5000"
|
|
bucket_name: $STORAGE_BUCKET_NAME|"pyinfra-test-bucket" # TODO Maybe refactor!
|
|
target_file_extension: $TARGET_FILE_EXTENSION|"pdf" # Defines type of file pulled from storage
|
|
|
|
probing_webserver:
|
|
host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address
|
|
port: $PROBING_WEBSERVER_PORT|8080 # Probe webserver port
|
|
mode: $PROBING_WEBSERVER_MODE|production # webserver mode: {development, production}
|
|
|
|
test:
|
|
minio:
|
|
s3:
|
|
endpoint: "http://127.0.0.1:9000"
|
|
access_key: root
|
|
secret_key: password
|
|
|
|
aws:
|
|
s3:
|
|
endpoint: https://s3.amazonaws.com
|
|
access_key: AKIA4QVP6D4LCDAGYGN2
|
|
secret_key: 8N6H1TUHTsbvW2qMAm7zZlJ63hMqjcXAsdN7TYED
|
|
|
|
azure:
|
|
azure_blob_storage:
|
|
connection_string: "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
|
|
|
|
bucket: "pyinfra-test-bucket"
|