Compare commits

...

11 Commits

Author SHA1 Message Date
Julius Unverfehrt
ec2ab890b8 swap case when the log is printed for env var parsing 2022-09-12 15:49:17 +02:00
Julius Unverfehrt
aaa02ea35e Merge branch 'master' of ssh://git.iqser.com:2222/rr/image-prediction into RED-5202-port-hotfixes 2022-09-12 15:28:39 +02:00
Julius Unverfehrt
5d87066b40 remove warning log for non existent non default env var 2022-09-12 15:25:10 +02:00
Julius Unverfehrt
23c61ef49e make env var parser discrete 2022-09-12 15:14:19 +02:00
Matthias Bisping
c1b9227035 fixed statefulness issue with os.environ in tests 2022-09-12 13:28:44 +02:00
Matthias Bisping
ad9c5657fe - Adapted response formatting logic for threshold maps passed via env vars.
- Added test for reading threshold maps and values from env vars.
2022-09-12 13:04:55 +02:00
Julius Unverfehrt
c60e8cd678 add parser for environment variables WIP 2022-09-12 11:38:01 +02:00
Julius Unverfehrt
101b71726c Add typehints, make custom page quotient breach function private since the intention of outsourcing it from build_image_info is to make it testable seperately 2022-09-12 09:52:33 +02:00
Julius Unverfehrt
04aee4e627 DotIndexable default get method exception made more specific 2022-09-12 09:25:59 +02:00
Julius Unverfehrt
4584e7ba66 RED-5202 port temporary broken image handling so the hotfix won't be lost by upgrading the service. A proper solution is still desirable (see RED-5148) 2022-09-12 08:55:05 +02:00
Julius Unverfehrt
5f99622646 RED-5202 add per class customizable max image to page quotient setting for signatures, default is 0.4. Can be overwritten by , set to null to use default value or set to value that should be used. 2022-09-12 08:47:02 +02:00
2 changed files with 51 additions and 48 deletions

View File

@ -4,7 +4,7 @@ import os
from functools import lru_cache
from operator import itemgetter
from funcy import filter, juxt, first, rest, compose
from funcy import first
from image_prediction.config import CONFIG
from image_prediction.exceptions import ParsingError
@ -20,45 +20,6 @@ class ResponseTransformer(Transformer):
return build_image_info(data)
def get_class_specific_min_image_to_page_quotient(label, table=None):
return get_class_specific_value(
"REL_IMAGE_SIZE", label, "min", CONFIG.filters.image_to_page_quotient.min, table=table
)
def get_class_specific_max_image_to_page_quotient(label, table=None):
return get_class_specific_value(
"REL_IMAGE_SIZE", label, "max", CONFIG.filters.image_to_page_quotient.max, table=table
)
def get_class_specific_min_image_width_to_height_quotient(label, table=None):
return get_class_specific_value(
"IMAGE_FORMAT", label, "min", CONFIG.filters.image_width_to_height_quotient.min, table=table
)
def get_class_specific_max_image_width_to_height_quotient(label, table=None):
return get_class_specific_value(
"IMAGE_FORMAT", label, "max", CONFIG.filters.image_width_to_height_quotient.max, table=table
)
def get_class_specific_min_classification_confidence(label, table=None):
return get_class_specific_value("CONFIDENCE", label, "min", CONFIG.filters.min_confidence, table=table)
def get_class_specific_value(prefix, label, bound, fallback_value, table=None):
def fallback():
logger.warning(f"Failed to resolve {bound} {prefix.lower().replace('_', '-')} value for class '{label}'.")
return fallback_value
assert bound in ["min", "max"]
threshold_map = parse_env_var(prefix, table=table) or {}
return threshold_map.get(label, {}).get(bound) or fallback()
def build_image_info(data: dict) -> dict:
def compute_geometric_quotient():
page_area_sqrt = math.sqrt(abs(page_width * page_height))
@ -128,19 +89,61 @@ def build_image_info(data: dict) -> dict:
return image_info
def get_class_specific_min_image_to_page_quotient(label, table=None):
return get_class_specific_value(
"REL_IMAGE_SIZE", label, "min", CONFIG.filters.image_to_page_quotient.min, table=table
)
def get_class_specific_max_image_to_page_quotient(label, table=None):
return get_class_specific_value(
"REL_IMAGE_SIZE", label, "max", CONFIG.filters.image_to_page_quotient.max, table=table
)
def get_class_specific_min_image_width_to_height_quotient(label, table=None):
return get_class_specific_value(
"IMAGE_FORMAT", label, "min", CONFIG.filters.image_width_to_height_quotient.min, table=table
)
def get_class_specific_max_image_width_to_height_quotient(label, table=None):
return get_class_specific_value(
"IMAGE_FORMAT", label, "max", CONFIG.filters.image_width_to_height_quotient.max, table=table
)
def get_class_specific_min_classification_confidence(label, table=None):
return get_class_specific_value("CONFIDENCE", label, "min", CONFIG.filters.min_confidence, table=table)
def get_class_specific_value(prefix, label, bound, fallback_value, table=None):
def fallback():
return fallback_value
def success():
threshold_map = parse_env_var(prefix, table=table) or {}
value = threshold_map.get(label, {}).get(bound)
if value:
logger.debug(f"Using class '{label}' specific {bound} {prefix.lower().replace('_', '-')} value.")
return value
assert bound in ["min", "max"]
return success() or fallback()
@lru_cache(maxsize=None)
def parse_env_var(prefix, table=None):
table = table or os.environ
head, tail = juxt(first, compose(list, rest))(filter(prefix, table))
if not head:
logger.warning(f"Found no environment variable with prefix '{prefix}'.")
elif tail:
logger.warning(f"Found multiple candidates for environment variable with prefix '{prefix}'.")
else:
head = first(filter(lambda s: s == prefix, table))
if head:
try:
return parse_env_var_value(table[head])
except ParsingError as err:
logger.warning(err)
else:
return None
def parse_env_var_value(env_var_value):

View File

@ -21,8 +21,8 @@ def label():
def page_quotient_threshold_map(label):
return frozendict(
{
"REL_IMAGE_SIZE_MAP": json.dumps({label: {"min": 0.1, "max": 0.2}}),
"IMAGE_FORMAT_MAP": json.dumps({label: {"min": 0.5, "max": 0.4}}),
"REL_IMAGE_SIZE": json.dumps({label: {"min": 0.1, "max": 0.2}}),
"IMAGE_FORMAT": json.dumps({label: {"min": 0.5, "max": 0.4}}),
"CONFIDENCE": json.dumps({label: {"min": 0.8}}),
}
)