From 13d4427c7801f0ac45797f30fe449f0a79f25cb1 Mon Sep 17 00:00:00 2001 From: Julius Unverfehrt Date: Mon, 12 Sep 2022 15:59:50 +0200 Subject: [PATCH] Pull request #33: RED-5202 port hotfixes Merge in RR/image-prediction from RED-5202-port-hotfixes to master Squashed commit of the following: commit 9674901235264de6b74d679fd39a52775ac4aee1 Merge: ec2ab89 9763d2c Author: Julius Unverfehrt Date: Mon Sep 12 15:55:58 2022 +0200 Merge remote-tracking branch 'origin' into RED-5202-port-hotfixes commit ec2ab890b8307942d147d6b8b236f6a3c1d0aebc Author: Julius Unverfehrt Date: Mon Sep 12 15:49:17 2022 +0200 swap case when the log is printed for env var parsing commit aaa02ea35e9c1b3b307116d7e3e32c93fd79ef5d Merge: 5d87066 521222e Author: Julius Unverfehrt Date: Mon Sep 12 15:28:39 2022 +0200 Merge branch 'master' of ssh://git.iqser.com:2222/rr/image-prediction into RED-5202-port-hotfixes commit 5d87066b40b28f919b1346f5e5396b46445b4e00 Author: Julius Unverfehrt Date: Mon Sep 12 15:25:01 2022 +0200 remove warning log for non existent non default env var commit 23c61ef49ef918b29952150d4a6e61b99d60ac64 Author: Julius Unverfehrt Date: Mon Sep 12 15:14:19 2022 +0200 make env var parser discrete commit c1b92270354c764861da0f7782348e9cd0725d76 Author: Matthias Bisping Date: Mon Sep 12 13:28:44 2022 +0200 fixed statefulness issue with os.environ in tests commit ad9c5657fe93079d5646ba2b70fa091e8d2daf76 Author: Matthias Bisping Date: Mon Sep 12 13:04:55 2022 +0200 - Adapted response formatting logic for threshold maps passed via env vars. - Added test for reading threshold maps and values from env vars. commit c60e8cd6781b8e0c3ec69ccd0a25375803de26f0 Author: Julius Unverfehrt Date: Mon Sep 12 11:38:01 2022 +0200 add parser for environment variables WIP commit 101b71726c697f30ec9298ba62d2203bd7da2efb Author: Julius Unverfehrt Date: Mon Sep 12 09:52:33 2022 +0200 Add typehints, make custom page quotient breach function private since the intention of outsourcing it from build_image_info is to make it testable seperately commit 04aee4e62781e78cd54c6d20e961dcd7bf1fc081 Author: Julius Unverfehrt Date: Mon Sep 12 09:25:59 2022 +0200 DotIndexable default get method exception made more specific commit 4584e7ba66400033dc5f1a38473b644eeb11e67c Author: Julius Unverfehrt Date: Mon Sep 12 08:55:05 2022 +0200 RED-5202 port temporary broken image handling so the hotfix won't be lost by upgrading the service. A proper solution is still desirable (see RED-5148) commit 5f99622646b3f6d3a842aebef91ff8e082072cd6 Author: Julius Unverfehrt Date: Mon Sep 12 08:47:02 2022 +0200 RED-5202 add per class customizable max image to page quotient setting for signatures, default is 0.4. Can be overwritten by , set to null to use default value or set to value that should be used. --- .../transformer/transformers/response.py | 83 ++++++++++--------- test/unit_tests/response_transformer_test.py | 4 +- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/image_prediction/transformer/transformers/response.py b/image_prediction/transformer/transformers/response.py index 7c2b326..378fe7b 100644 --- a/image_prediction/transformer/transformers/response.py +++ b/image_prediction/transformer/transformers/response.py @@ -20,45 +20,6 @@ class ResponseTransformer(Transformer): return build_image_info(data) -def get_class_specific_min_image_to_page_quotient(label, table=None): - return get_class_specific_value( - "REL_IMAGE_SIZE", label, "min", CONFIG.filters.image_to_page_quotient.min, table=table - ) - - -def get_class_specific_max_image_to_page_quotient(label, table=None): - return get_class_specific_value( - "REL_IMAGE_SIZE", label, "max", CONFIG.filters.image_to_page_quotient.max, table=table - ) - - -def get_class_specific_min_image_width_to_height_quotient(label, table=None): - return get_class_specific_value( - "IMAGE_FORMAT", label, "min", CONFIG.filters.image_width_to_height_quotient.min, table=table - ) - - -def get_class_specific_max_image_width_to_height_quotient(label, table=None): - return get_class_specific_value( - "IMAGE_FORMAT", label, "max", CONFIG.filters.image_width_to_height_quotient.max, table=table - ) - - -def get_class_specific_min_classification_confidence(label, table=None): - return get_class_specific_value("CONFIDENCE", label, "min", CONFIG.filters.min_confidence, table=table) - - -def get_class_specific_value(prefix, label, bound, fallback_value, table=None): - def fallback(): - logger.warning(f"Failed to resolve {bound} {prefix.lower().replace('_', '-')} value for class '{label}'.") - return fallback_value - - assert bound in ["min", "max"] - - threshold_map = parse_env_var(prefix, table=table) or {} - return threshold_map.get(label, {}).get(bound) or fallback() - - def build_image_info(data: dict) -> dict: def compute_geometric_quotient(): page_area_sqrt = math.sqrt(abs(page_width * page_height)) @@ -128,6 +89,50 @@ def build_image_info(data: dict) -> dict: return image_info +def get_class_specific_min_image_to_page_quotient(label, table=None): + return get_class_specific_value( + "REL_IMAGE_SIZE", label, "min", CONFIG.filters.image_to_page_quotient.min, table=table + ) + + +def get_class_specific_max_image_to_page_quotient(label, table=None): + return get_class_specific_value( + "REL_IMAGE_SIZE", label, "max", CONFIG.filters.image_to_page_quotient.max, table=table + ) + + +def get_class_specific_min_image_width_to_height_quotient(label, table=None): + return get_class_specific_value( + "IMAGE_FORMAT", label, "min", CONFIG.filters.image_width_to_height_quotient.min, table=table + ) + + +def get_class_specific_max_image_width_to_height_quotient(label, table=None): + return get_class_specific_value( + "IMAGE_FORMAT", label, "max", CONFIG.filters.image_width_to_height_quotient.max, table=table + ) + + +def get_class_specific_min_classification_confidence(label, table=None): + return get_class_specific_value("CONFIDENCE", label, "min", CONFIG.filters.min_confidence, table=table) + + +def get_class_specific_value(prefix, label, bound, fallback_value, table=None): + def fallback(): + return fallback_value + + def success(): + threshold_map = parse_env_var(prefix, table=table) or {} + value = threshold_map.get(label, {}).get(bound) + if value: + logger.debug(f"Using class '{label}' specific {bound} {prefix.lower().replace('_', '-')} value.") + return value + + assert bound in ["min", "max"] + + return success() or fallback() + + @lru_cache(maxsize=None) def parse_env_var(prefix, table=None): table = table or os.environ diff --git a/test/unit_tests/response_transformer_test.py b/test/unit_tests/response_transformer_test.py index ac8d822..c93e1af 100644 --- a/test/unit_tests/response_transformer_test.py +++ b/test/unit_tests/response_transformer_test.py @@ -21,8 +21,8 @@ def label(): def page_quotient_threshold_map(label): return frozendict( { - "REL_IMAGE_SIZE_MAP": json.dumps({label: {"min": 0.1, "max": 0.2}}), - "IMAGE_FORMAT_MAP": json.dumps({label: {"min": 0.5, "max": 0.4}}), + "REL_IMAGE_SIZE": json.dumps({label: {"min": 0.1, "max": 0.2}}), + "IMAGE_FORMAT": json.dumps({label: {"min": 0.5, "max": 0.4}}), "CONFIDENCE": json.dumps({label: {"min": 0.8}}), } )