55be1c345b
Merge in RR/pyinfra from add-storage-save-response to master
Squashed commit of the following:
commit cb96b7943c388f020d11e97e7e1f8402efa2dd12
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Mon Feb 28 16:06:29 2022 +0100
improved config
commit 607e938f16a39215c81c88bb76377bcc90282011
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Mon Feb 28 15:57:21 2022 +0100
response save on storage logic added
commit 7b66bcaafe32e80a64ad09ef424c28fe1f73cd23
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Mon Feb 28 15:22:19 2022 +0100
tidy up & black
commit 15a5687bac5bac599d162c86f5cec2cd945c5039
Author: Julius Unverfehrt <Julius.Unverfehrt@iqser.com>
Date: Mon Feb 28 15:20:50 2022 +0100
response save on storage logic added WIP
48 lines
2.1 KiB
YAML
Executable File
48 lines
2.1 KiB
YAML
Executable File
service:
|
|
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger
|
|
response:
|
|
save: True # file-to-storage upload
|
|
extension: $RESPONSE_FILE_EXTENSION|".NER_ENTITIES.json.gz" # {.OBJECTS.json.gz | .NER_ENTITIES.json.gz}
|
|
|
|
probing_webserver:
|
|
host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address
|
|
port: $PROBING_WEBSERVER_PORT|8080 # Probe webserver port
|
|
mode: $PROBING_WEBSERVER_MODE|production # webserver mode: {development, production}
|
|
|
|
rabbitmq:
|
|
host: $RABBITMQ_HOST|localhost # RabbitMQ host address
|
|
port: $RABBITMQ_PORT|5672 # RabbitMQ host port
|
|
user: $RABBITMQ_USERNAME|user # RabbitMQ username
|
|
password: $RABBITMQ_PASSWORD|bitnami # RabbitMQ password
|
|
heartbeat: $RABBITMQ_HEARTBEAT|7200 # Controls AMQP heartbeat timeout in seconds
|
|
|
|
queues:
|
|
input: $REQUEST_QUEUE|request_queue # Requests to service
|
|
output: $RESPONSE_QUEUE|response_queue # Responses by service
|
|
dead_letter: $DEAD_LETTER_QUEUE|dead_letter_queue # Messages that failed to process
|
|
|
|
prefetch_count: 1
|
|
|
|
callback:
|
|
|
|
retry: # Controls retry behaviour for messages the processing of which failed
|
|
# TODO: check if this actually works
|
|
enabled: $RETRY|False # Toggles retry behaviour
|
|
max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue
|
|
|
|
analysis_endpoint: $ANALYSIS_ENDPOINT|"http://127.0.0.1:5000"
|
|
|
|
storage:
|
|
|
|
backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure}
|
|
bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from
|
|
target_file_extension: $TARGET_FILE_EXTENSION|".TEXT.json.gz" # {.TEXT.json.gz | .ORIGIN.pdf.gz} Defines type of file to pull from storage
|
|
|
|
s3:
|
|
endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000"
|
|
access_key: $STORAGE_KEY|root
|
|
secret_key: $STORAGE_SECRET|password
|
|
|
|
azure:
|
|
connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
|