From 251fea4094062a72f5f0b1f8a54f959a1d7309ec Mon Sep 17 00:00:00 2001 From: Julius Unverfehrt Date: Tue, 15 Mar 2022 15:02:25 +0100 Subject: [PATCH] Adjusted readme and config to the actual changes. --- README.md | 186 +++++++++++++++++++--------------------------------- config.yaml | 14 ---- 2 files changed, 66 insertions(+), 134 deletions(-) diff --git a/README.md b/README.md index 5ecaa77..33f8e90 100755 --- a/README.md +++ b/README.md @@ -6,154 +6,100 @@ The Infrastructure expects to be deployed in the same Pod / local environment as A configuration is located in `/config.yaml`. All relevant variables can be configured via exporting environment variables. -| Environment Variable | Default | Description | -|-------------------------------|--------------------------------|--------------------------------------------------------------------------------------------------| -| _service_ | | | -| LOGGING_LEVEL_ROOT | DEBUG | Logging level for service logger | -| RESPONSE_TYPE | "stream" | Whether the analysis response is stored as file on storage or sent as stream: "file" or "stream" | -| RESPONSE_FILE_EXTENSION | ".NER_ENTITIES.json.gz" | Extension to the file that stores the analyzed response on storage | -| _probing_webserver_ | | | -| PROBING_WEBSERVER_HOST | "0.0.0.0" | Probe webserver address | -| PROBING_WEBSERVER_PORT | 8080 | Probe webserver port | -| PROBING_WEBSERVER_MODE | production | Webserver mode: {development, production} | -| _rabbitmq_ | | | -| RABBITMQ_HOST | localhost | RabbitMQ host address | -| RABBITMQ_PORT | 5672 | RabbitMQ host port | -| RABBITMQ_USERNAME | user | RabbitMQ username | -| RABBITMQ_PASSWORD | bitnami | RabbitMQ password | -| RABBITMQ_HEARTBEAT | 7200 | Controls AMQP heartbeat timeout in seconds | -| _queues_ | | | -| REQUEST_QUEUE | request_queue | Requests to service | -| RESPONSE_QUEUE | response_queue | Responses by service | -| DEAD_LETTER_QUEUE | dead_letter_queue | Messages that failed to process | -| _callback_ | | | -| RETRY | False | Toggles retry behaviour | -| MAX_ATTEMPTS | 3 | Number of times a message may fail before being published to dead letter queue | -| ANALYSIS_ENDPOINT | "http://127.0.0.1:5000" | | -| _storage_ | | | -| STORAGE_BACKEND | s3 | The type of storage to use {s3, azure} | -| STORAGE_BUCKET | "pyinfra-test-bucket" | The bucket / container to pull files specified in queue requests from | -| TARGET_FILE_EXTENSION | ".TEXT.json.gz" | Defines type of file to pull from storage: .TEXT.json.gz or .ORIGIN.pdf.gz | -| STORAGE_ENDPOINT | "http://127.0.0.1:9000" | | -| STORAGE_KEY | | | -| STORAGE_SECRET | | | -| STORAGE_AZURECONNECTIONSTRING | "DefaultEndpointsProtocol=..." | | +| Environment Variable | Default | Description | +|-------------------------------|--------------------------------|-----------------------------------------------------------------------| +| LOGGING_LEVEL_ROOT | DEBUG | Logging level for service logger | +| PROBING_WEBSERVER_HOST | "0.0.0.0" | Probe webserver address | +| PROBING_WEBSERVER_PORT | 8080 | Probe webserver port | +| PROBING_WEBSERVER_MODE | production | Webserver mode: {development, production} | +| RABBITMQ_HOST | localhost | RabbitMQ host address | +| RABBITMQ_PORT | 5672 | RabbitMQ host port | +| RABBITMQ_USERNAME | user | RabbitMQ username | +| RABBITMQ_PASSWORD | bitnami | RabbitMQ password | +| RABBITMQ_HEARTBEAT | 7200 | Controls AMQP heartbeat timeout in seconds | +| REQUEST_QUEUE | request_queue | Requests to service | +| RESPONSE_QUEUE | response_queue | Responses by service | +| DEAD_LETTER_QUEUE | dead_letter_queue | Messages that failed to process | +| ANALYSIS_ENDPOINT | "http://127.0.0.1:5000" | Endpoint for analysis container | +| STORAGE_BACKEND | s3 | The type of storage to use {s3, azure} | +| STORAGE_BUCKET | "pyinfra-test-bucket" | The bucket / container to pull files specified in queue requests from | +| STORAGE_ENDPOINT | "http://127.0.0.1:9000" | Endpoint for s3 storage | +| STORAGE_KEY | root | User for s3 storage | +| STORAGE_SECRET | password | Password for s3 storage | +| STORAGE_AZURECONNECTIONSTRING | "DefaultEndpointsProtocol=..." | Connection string for Azure storage | ## Response Format -### RESPONSE_AS_FILE == False - -Response-Format: +### Expected AMQP input message: ```json { - "dossierId": "klaus", - "fileId": "1a7fd8ac0da7656a487b68f89188be82", - "imageMetadata": ANALYSIS_DATA + "dossierId": "", + "fileId": "", + "targetFileExtension": "", + "responseFileExtension": "" } ``` -Response-example for image-prediction +Optionally, the input message can contain a field with the key `"operations"`. + +### AMQP output message: ```json { - "dossierId": "klaus", - "fileId": "1a7fd8ac0da7656a487b68f89188be82", - "imageMetadata": [ - { - "classification": { - "label": "logo", - "probabilities": { - "formula": 0.0, - "logo": 1.0, - "other": 0.0, - "signature": 0.0 - } - }, - "filters": { - "allPassed": true, - "geometry": { - "imageFormat": { - "quotient": 1.570791527313267, - "tooTall": false, - "tooWide": false - }, - "imageSize": { - "quotient": 0.19059804229011604, - "tooLarge": false, - "tooSmall": false - } - }, - "probability": { - "unconfident": false - } - }, - "geometry": { - "height": 107.63999999999999, - "width": 169.08000000000004 - }, - "position": { - "pageNumber": 1, - "x1": 213.12, - "x2": 382.20000000000005, - "y1": 568.7604, - "y2": 676.4004 - } - } - ] + "dossierId": "", + "fileId": "", + ... } ``` -### RESPONSE_AS_FILE == True - -Creates a respone file on the request storage, named `dossier_Id / file_Id + RESPONSE_FILE_EXTENSION` with the `ANALYSIS_DATA` as content. - ## Development -### Local Setup +Either run `src/serve.py` or the built Docker image. -You can run the infrastructure either as module via. `src/serve.py` or as Dockercontainer simulating the kubernetes environment +### Setup -1. Install module / build docker image +Install module. - ```bash - pip install -e . - pip install -r requirements.txt - ``` +```bash + pip install -e . + pip install -r requirements.txt +``` - ```bash - docker build -f Dockerfile -t pyinfra . - ``` +or build docker image. -2. Run rabbitmq & minio +```bash +docker build -f Dockerfile -t pyinfra . +``` - ```bash - docker-compose up - ``` +### Usage -3. Run module +**Shell 1:** Start a MinIO and a RabbitMQ docker container. - ```bash - python src/serve.py - ``` - OR as container: - ```bash - docker run --net=host pyinfra - ``` - -Start your prediction container for example ner-prediction or image-prediction (follow their corresponding README for -building the container). +```bash +docker-compose up +``` -To put a file on the queue do: +**Shell 2:** Add files to the local minio storage. - python src/manage_minio.py add --file path/to/file dossierID +```bash +python scripts/manage_minio.py add -d path/to/a/folder/with/PDFs +``` -To start mock: +**Shell 2:** Run pyinfra-server. - python src/mock_client.py +```bash +python src/serve.py +``` +or as container: -### Hints: -When stopping the docker-compose up, use docker-compose down to remove containers created by up. +```bash +docker run --net=host pyinfra +``` -If uploaded files are stuck, clean the minio storage by using ```python src/manage_minio.py purge``` or delete local -minio data folder in pyinfra with ```sudo rm -rf data``` \ No newline at end of file +**Shell 3:** Run analysis-container. + +**Shell 4:** Start a client that sends requests to process PDFs from the MinIO store and annotates these PDFs according to the service responses. +```bash +python scripts/mock_client.py +``` diff --git a/config.yaml b/config.yaml index cfc6aed..ceb12a0 100755 --- a/config.yaml +++ b/config.yaml @@ -1,9 +1,5 @@ service: logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger - response: - type: $RESPONSE_TYPE|"file" # Whether the analysis response is stored as file on storage or sent as stream -# extension: $RESPONSE_FILE_EXTENSION|"IMAGE_INFO.json.gz" # {.IMAGE_INFO.json.gz | .NER_ENTITIES.json.gz} -# key: $RESPONSE_KEY|"imageMetadata" # the key of the result {result, imageMetadata} probing_webserver: host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address @@ -22,22 +18,12 @@ rabbitmq: output: $RESPONSE_QUEUE|response_queue # Responses by service dead_letter: $DEAD_LETTER_QUEUE|dead_letter_queue # Messages that failed to process - prefetch_count: 1 - callback: - - retry: # Controls retry behaviour for messages the processing of which failed - # TODO: check if this actually works - enabled: $RETRY|False # Toggles retry behaviour - max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue - analysis_endpoint: $ANALYSIS_ENDPOINT|"http://127.0.0.1:5000" storage: - backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure} bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from - # TODO: Caller should specify exact file name, including extension! s3: endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000"