From 6427a32072530864008c66650eeeb030697df9fb Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 18 Mar 2022 21:12:49 +0100 Subject: [PATCH 01/15] raise specific exceptions during value retrieval and processing This makes it easier to determine what exactly went wrong and thus build better error handling in a later commit. Tests were adapted accordingly to assert that only those specific errors are raised instead of the plain ValueErrors. --- configurations/values.py | 72 ++++++++++++++++++++++++++++------------ tests/test_values.py | 60 ++++++++++++++++++++------------- 2 files changed, 87 insertions(+), 45 deletions(-) diff --git a/configurations/values.py b/configurations/values.py index 8eb3bbc..215091d 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -11,6 +11,31 @@ from django.utils.module_loading import import_string from .utils import getargspec +class ValueRetrievalError(ValueError): + """ + Exception is raised when errors occur during the retrieval of a dynamic Value. + This can happen when the environment variable corresponding to the value is not defined. + """ + + def __init__(self, value_instance: 'Value', error_msg: str) -> None: + super().__init__(error_msg) + self.value_instance = value_instance + self.error_msg = error_msg + + +class ValueProcessingError(ValueError): + """ + Exception that is raised when a dynamic Value failed to be processed after retrieval. + Processing could be i.e. converting from string to a native datatype but also validation. + """ + + def __init__(self, value_instance: 'Value', str_value: str, error_msg: str) -> None: + super().__init__(error_msg) + self.value_instance = value_instance + self.str_value = str_value + self.error_msg = error_msg + + def setup_value(target, name, value): actual_value = value.setup(name) # overwriting the original Value class with the result @@ -102,9 +127,9 @@ class Value: if full_environ_name in os.environ: value = self.to_python(os.environ[full_environ_name]) elif self.environ_required: - raise ValueError('Value {0!r} is required to be set as the ' - 'environment variable {1!r}' - .format(name, full_environ_name)) + raise ValueRetrievalError(self, 'Value {0!r} is required to be set as the ' + 'environment variable {1!r}' + .format(name, full_environ_name)) self.value = value return value @@ -128,8 +153,8 @@ class BooleanValue(Value): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.default not in (True, False): - raise ValueError('Default value {0!r} is not a ' - 'boolean value'.format(self.default)) + raise ImproperlyConfigured('Default value {0!r} is not a ' + 'boolean value'.format(self.default)) def to_python(self, value): normalized_value = value.strip().lower() @@ -138,8 +163,8 @@ class BooleanValue(Value): elif normalized_value in self.false_values: return False else: - raise ValueError('Cannot interpret ' - 'boolean value {0!r}'.format(value)) + raise ValueProcessingError(self, value, 'Cannot interpret ' + 'boolean value {0!r}'.format(value)) class CastingMixin: @@ -159,7 +184,7 @@ class CastingMixin: else: error = 'Cannot use caster of {0} ({1!r})'.format(self, self.caster) - raise ValueError(error) + raise ImproperlyConfigured(error) try: arg_names = getargspec(self._caster)[0] self._params = {name: kwargs[name] for name in arg_names if name in kwargs} @@ -173,7 +198,7 @@ class CastingMixin: else: return self._caster(value) except self.exception: - raise ValueError(self.message.format(value)) + raise ValueProcessingError(self, value, self.message.format(value)) class IntegerValue(CastingMixin, Value): @@ -185,7 +210,7 @@ class PositiveIntegerValue(IntegerValue): def to_python(self, value): int_value = super().to_python(value) if int_value < 0: - raise ValueError(self.message.format(value)) + raise ValueProcessingError(self, value, self.message.format(value)) return int_value @@ -231,7 +256,7 @@ class SequenceValue(Value): try: converted_values.append(self.converter(value)) except (TypeError, ValueError): - raise ValueError(self.message.format(value, value)) + raise ValueProcessingError(self, self.separator.join(sequence), self.message.format(value, value)) return self.sequence_type(converted_values) def to_python(self, value): @@ -296,7 +321,7 @@ class BackendsValue(ListValue): try: import_string(value) except ImportError as err: - raise ValueError(err).with_traceback(sys.exc_info()[2]) + raise ValueProcessingError(self, value, str(err)).with_traceback(sys.exc_info()[2]) return value @@ -331,9 +356,9 @@ class DictValue(Value): try: evaled_value = ast.literal_eval(value) except ValueError: - raise ValueError(self.message.format(value)) + raise ValueProcessingError(self, value, self.message.format(value)) if not isinstance(evaled_value, dict): - raise ValueError(self.message.format(value)) + raise ValueProcessingError(self, value, self.message.format(value)) return evaled_value @@ -350,16 +375,19 @@ class ValidationMixin: elif callable(self.validator): self._validator = self.validator else: - raise ValueError('Cannot use validator of ' - '{0} ({1!r})'.format(self, self.validator)) + raise ImproperlyConfigured('Cannot use validator of ' + '{0} ({1!r})'.format(self, self.validator)) if self.default: - self.to_python(self.default) + try: + self.to_python(self.default) + except ValueProcessingError as e: + raise ImproperlyConfigured(e.error_msg) def to_python(self, value): try: self._validator(value) except ValidationError: - raise ValueError(self.message.format(value)) + raise ValueProcessingError(self, value, self.message.format(value)) else: return value @@ -397,7 +425,7 @@ class PathValue(Value): value = super().setup(name) value = os.path.expanduser(value) if self.check_exists and not os.path.exists(value): - raise ValueError('Path {0!r} does not exist.'.format(value)) + raise ValueProcessingError(self, value, 'Path {0!r} does not exist.'.format(value)) return os.path.abspath(value) @@ -408,13 +436,13 @@ class SecretValue(Value): kwargs['environ_required'] = True super().__init__(*args, **kwargs) if self.default is not None: - raise ValueError('Secret values are only allowed to ' - 'be set as environment variables') + raise ImproperlyConfigured('Secret values are only allowed to ' + 'be set as environment variables') def setup(self, name): value = super().setup(name) if not value: - raise ValueError('Secret value {0!r} is not set'.format(name)) + raise ValueRetrievalError(self, 'Secret value {0!r} is not set'.format(name)) return value diff --git a/tests/test_values.py b/tests/test_values.py index 2547e50..6eb1ca4 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -15,8 +15,8 @@ from configurations.values import (Value, BooleanValue, IntegerValue, RegexValue, PathValue, SecretValue, DatabaseURLValue, EmailURLValue, CacheURLValue, BackendsValue, - CastingMixin, SearchURLValue, - setup_value, PositiveIntegerValue) + SearchURLValue, PositiveIntegerValue, CastingMixin, + setup_value, ValueRetrievalError, ValueProcessingError) @contextmanager @@ -58,6 +58,19 @@ class ValueTests(TestCase): self.assertEqual(repr(value), repr('override')) + def test_environ_required(self): + for ValueClass in (Value, BooleanValue, IntegerValue, + FloatValue, DecimalValue, ListValue, + TupleValue, SingleNestedTupleValue, + SingleNestedListValue, SetValue, + DictValue, URLValue, EmailValue, IPValue, + RegexValue, PathValue, SecretValue, + DatabaseURLValue, EmailURLValue, + CacheURLValue, BackendsValue, + SearchURLValue, PositiveIntegerValue): + value = ValueClass(environ_required=True) + self.assertRaises(ValueRetrievalError, value.setup, "TEST") + def test_value_truthy(self): value = Value('default') self.assertTrue(bool(value)) @@ -110,7 +123,7 @@ class ValueTests(TestCase): self.assertTrue(bool(value.setup('TEST'))) def test_boolean_values_faulty(self): - self.assertRaises(ValueError, BooleanValue, 'false') + self.assertRaises(ImproperlyConfigured, BooleanValue, 'false') def test_boolean_values_false(self): value = BooleanValue(True) @@ -121,7 +134,7 @@ class ValueTests(TestCase): def test_boolean_values_nonboolean(self): value = BooleanValue(True) with env(DJANGO_TEST='nonboolean'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_boolean_values_assign_false_to_another_booleanvalue(self): value1 = BooleanValue(False) @@ -134,30 +147,30 @@ class ValueTests(TestCase): with env(DJANGO_TEST='2'): self.assertEqual(value.setup('TEST'), 2) with env(DJANGO_TEST='noninteger'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_positive_integer_values(self): value = PositiveIntegerValue(1) with env(DJANGO_TEST='2'): self.assertEqual(value.setup('TEST'), 2) with env(DJANGO_TEST='noninteger'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') with env(DJANGO_TEST='-1'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_float_values(self): value = FloatValue(1.0) with env(DJANGO_TEST='2.0'): self.assertEqual(value.setup('TEST'), 2.0) with env(DJANGO_TEST='noninteger'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_decimal_values(self): value = DecimalValue(decimal.Decimal(1)) with env(DJANGO_TEST='2'): self.assertEqual(value.setup('TEST'), decimal.Decimal(2)) with env(DJANGO_TEST='nondecimal'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_failing_caster(self): self.assertRaises(ImproperlyConfigured, FailingCasterValue) @@ -194,7 +207,7 @@ class ValueTests(TestCase): def test_list_values_converter_exception(self): value = ListValue(converter=int) with env(DJANGO_TEST='2,b'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_tuple_values_default(self): value = TupleValue() @@ -292,21 +305,21 @@ class ValueTests(TestCase): with env(DJANGO_TEST=''): self.assertEqual(value.setup('TEST'), {}) with env(DJANGO_TEST='spam'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_email_values(self): value = EmailValue('spam@eg.gs') with env(DJANGO_TEST='spam@sp.am'): self.assertEqual(value.setup('TEST'), 'spam@sp.am') with env(DJANGO_TEST='spam'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_url_values(self): value = URLValue('http://eggs.spam') with env(DJANGO_TEST='http://spam.eggs'): self.assertEqual(value.setup('TEST'), 'http://spam.eggs') with env(DJANGO_TEST='httb://spam.eggs'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_url_values_with_no_default(self): value = URLValue() # no default @@ -314,7 +327,7 @@ class ValueTests(TestCase): self.assertEqual(value.setup('TEST'), 'http://spam.eggs') def test_url_values_with_wrong_default(self): - self.assertRaises(ValueError, URLValue, 'httb://spam.eggs') + self.assertRaises(ImproperlyConfigured, URLValue, 'httb://spam.eggs') def test_ip_values(self): value = IPValue('0.0.0.0') @@ -323,14 +336,14 @@ class ValueTests(TestCase): with env(DJANGO_TEST='::1'): self.assertEqual(value.setup('TEST'), '::1') with env(DJANGO_TEST='spam.eggs'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_regex_values(self): value = RegexValue('000--000', regex=r'\d+--\d+') with env(DJANGO_TEST='123--456'): self.assertEqual(value.setup('TEST'), '123--456') with env(DJANGO_TEST='123456'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_path_values_with_check(self): value = PathValue() @@ -339,7 +352,7 @@ class ValueTests(TestCase): with env(DJANGO_TEST='~/'): self.assertEqual(value.setup('TEST'), os.path.expanduser('~')) with env(DJANGO_TEST='/does/not/exist'): - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_path_values_no_check(self): value = PathValue(check_exists=False) @@ -354,17 +367,17 @@ class ValueTests(TestCase): def test_secret_value(self): # no default allowed, only environment values are - self.assertRaises(ValueError, SecretValue, 'default') + self.assertRaises(ImproperlyConfigured, SecretValue, 'default') value = SecretValue() - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueRetrievalError, value.setup, 'TEST') with env(DJANGO_SECRET_KEY='123'): self.assertEqual(value.setup('SECRET_KEY'), '123') value = SecretValue(environ_name='FACEBOOK_API_SECRET', environ_prefix=None, late_binding=True) - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueRetrievalError, value.setup, 'TEST') with env(FACEBOOK_API_SECRET='123'): self.assertEqual(value.setup('TEST'), '123') @@ -424,7 +437,7 @@ class ValueTests(TestCase): 'EMAIL_USE_SSL': False, 'EMAIL_USE_TLS': False}) with env(EMAIL_URL='smtps://user@domain.com:password@smtp.example.com:wrong'): # noqa: E501 - self.assertRaises(ValueError, value.setup, 'TEST') + self.assertRaises(ValueProcessingError, value.setup, 'TEST') def test_cache_url_value(self): cache_setting = { @@ -445,7 +458,7 @@ class ValueTests(TestCase): value.setup('TEST') self.assertEqual(cm.exception.args[0], 'Unknown backend: "wrong"') with env(CACHE_URL='redis://user@host:port/1'): - with self.assertRaises(ValueError) as cm: + with self.assertRaises(ValueProcessingError) as cm: value.setup('TEST') self.assertEqual( cm.exception.args[0], @@ -468,7 +481,7 @@ class ValueTests(TestCase): self.assertEqual(value.setup('TEST'), backends) backends = ['non.existing.Backend'] - self.assertRaises(ValueError, BackendsValue, backends) + self.assertRaises(ValueProcessingError, BackendsValue, backends) def test_tuple_value(self): value = TupleValue(None) @@ -503,6 +516,7 @@ class ValueTests(TestCase): 'EMAIL_HOST_PASSWORD': 'password', 'EMAIL_HOST_USER': 'user@domain.com', 'EMAIL_PORT': 587, + 'EMAIL_TIMEOUT': None, 'EMAIL_USE_SSL': False, 'EMAIL_USE_TLS': True }) From 0ec0a4c7ed6cef713b36bfa450198a1ad192bc56 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 18 Mar 2022 21:16:27 +0100 Subject: [PATCH 02/15] fix raised error on required BooleanValue and invalid default Previously and ImproperlyConfigured error was always raised when an invalid default configured for the BooleanValue class. This behavior is unnecessary if the default is never used because the value is marked as environ_required. The check has thus been adapted to ignore invalid defaults in that case. --- configurations/values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/values.py b/configurations/values.py index 215091d..6642c95 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -152,7 +152,7 @@ class BooleanValue(Value): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.default not in (True, False): + if not self.environ_required and self.default not in (True, False): raise ImproperlyConfigured('Default value {0!r} is not a ' 'boolean value'.format(self.default)) From 7e05c085be52244c7c6c6606b0da0892365c38c6 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 18 Mar 2022 21:56:21 +0100 Subject: [PATCH 03/15] add custom error handling for own exceptions This introduces a function that wraps django entry points with an error handler so that our own exception can be pretty-printed to an end-user without a large stack-trace. The defined error handler has also been applied to the existing management, wsgi, asgi and fastcgi entry point definitions. --- configurations/asgi.py | 5 ++++- configurations/error_handling.py | 20 ++++++++++++++++++++ configurations/fastcgi.py | 5 ++++- configurations/management.py | 8 ++++++-- configurations/wsgi.py | 5 ++++- 5 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 configurations/error_handling.py diff --git a/configurations/asgi.py b/configurations/asgi.py index da9401b..7644f0b 100644 --- a/configurations/asgi.py +++ b/configurations/asgi.py @@ -1,8 +1,11 @@ from . import importer +from .error_handling import with_error_handler importer.install() -from django.core.asgi import get_asgi_application # noqa: E402 +from django.core.asgi import get_asgi_application as dj_get_asgi_application # noqa: E402 + +get_asgi_application = with_error_handler(dj_get_asgi_application) # this is just for the crazy ones application = get_asgi_application() diff --git a/configurations/error_handling.py b/configurations/error_handling.py new file mode 100644 index 0000000..bb1de79 --- /dev/null +++ b/configurations/error_handling.py @@ -0,0 +1,20 @@ +from typing import * +from functools import wraps +import sys + +from .values import ValueRetrievalError, ValueProcessingError + + +def with_error_handler(callee: Callable) -> Callable: + """ + A decorator which is designed to wrap django entry points with an error handler so that django-configuration + originated errors can be caught and rendered to the user in a readable format. + """ + @wraps(callee) + def wrapper(*args, **kwargs): + try: + return callee(*args, **kwargs) + except (ValueRetrievalError, ValueProcessingError) as e: + print(e, file=sys.stderr) + + return wrapper diff --git a/configurations/fastcgi.py b/configurations/fastcgi.py index 5f654de..476bbaa 100644 --- a/configurations/fastcgi.py +++ b/configurations/fastcgi.py @@ -1,5 +1,8 @@ from . import importer +from .error_handling import with_error_handler importer.install() -from django.core.servers.fastcgi import runfastcgi # noqa +from django.core.servers.fastcgi import dj_runfastcgi # noqa + +runfastcgi = with_error_handler(dj_runfastcgi) diff --git a/configurations/management.py b/configurations/management.py index e718ef5..b175c8b 100644 --- a/configurations/management.py +++ b/configurations/management.py @@ -1,6 +1,10 @@ from . import importer +from .error_handling import with_error_handler importer.install(check_options=True) -from django.core.management import (execute_from_command_line, # noqa - call_command) +from django.core.management import (execute_from_command_line as dj_execute_from_command_line, # noqa + call_command as dj_call_command) + +execute_from_command_line = with_error_handler(dj_execute_from_command_line) +call_command = with_error_handler(dj_call_command) diff --git a/configurations/wsgi.py b/configurations/wsgi.py index ea157a3..af5f21a 100644 --- a/configurations/wsgi.py +++ b/configurations/wsgi.py @@ -1,8 +1,11 @@ from . import importer +from .error_handling import with_error_handler importer.install() -from django.core.wsgi import get_wsgi_application # noqa: E402 +from django.core.wsgi import get_wsgi_application as dj_get_wsgi_application # noqa: E402 + +get_wsgi_application = with_error_handler(dj_get_wsgi_application) # this is just for the crazy ones application = get_wsgi_application() From a33b77ad3c0b90736a24ca298ea0b101e2dab94d Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Fri, 18 Mar 2022 22:10:49 +0100 Subject: [PATCH 04/15] pretty-print configuration errors in red on ttys --- configurations/error_handling.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/configurations/error_handling.py b/configurations/error_handling.py index bb1de79..c863ce8 100644 --- a/configurations/error_handling.py +++ b/configurations/error_handling.py @@ -1,10 +1,17 @@ from typing import * from functools import wraps import sys +import os from .values import ValueRetrievalError, ValueProcessingError +class TermStyles: + BOLD = "\033[1m" + RED = "\033[91m" + END = "\033[0m" + + def with_error_handler(callee: Callable) -> Callable: """ A decorator which is designed to wrap django entry points with an error handler so that django-configuration @@ -15,6 +22,9 @@ def with_error_handler(callee: Callable) -> Callable: try: return callee(*args, **kwargs) except (ValueRetrievalError, ValueProcessingError) as e: - print(e, file=sys.stderr) + msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) \ + if os.isatty(sys.stderr.fileno()) \ + else str(e) + print(msg, file=sys.stderr) return wrapper From ece1521acda4f2986634bf26172ec79f2999cd50 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Sat, 19 Mar 2022 23:11:53 +0100 Subject: [PATCH 05/15] put error handling and error definitions in one file named errors.py --- configurations/asgi.py | 2 +- configurations/error_handling.py | 30 ------------------ configurations/errors.py | 53 ++++++++++++++++++++++++++++++++ configurations/fastcgi.py | 2 +- configurations/management.py | 2 +- configurations/values.py | 26 +--------------- configurations/wsgi.py | 2 +- tests/test_values.py | 3 +- 8 files changed, 60 insertions(+), 60 deletions(-) delete mode 100644 configurations/error_handling.py create mode 100644 configurations/errors.py diff --git a/configurations/asgi.py b/configurations/asgi.py index 7644f0b..bd744e1 100644 --- a/configurations/asgi.py +++ b/configurations/asgi.py @@ -1,5 +1,5 @@ from . import importer -from .error_handling import with_error_handler +from .errors import with_error_handler importer.install() diff --git a/configurations/error_handling.py b/configurations/error_handling.py deleted file mode 100644 index c863ce8..0000000 --- a/configurations/error_handling.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import * -from functools import wraps -import sys -import os - -from .values import ValueRetrievalError, ValueProcessingError - - -class TermStyles: - BOLD = "\033[1m" - RED = "\033[91m" - END = "\033[0m" - - -def with_error_handler(callee: Callable) -> Callable: - """ - A decorator which is designed to wrap django entry points with an error handler so that django-configuration - originated errors can be caught and rendered to the user in a readable format. - """ - @wraps(callee) - def wrapper(*args, **kwargs): - try: - return callee(*args, **kwargs) - except (ValueRetrievalError, ValueProcessingError) as e: - msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) \ - if os.isatty(sys.stderr.fileno()) \ - else str(e) - print(msg, file=sys.stderr) - - return wrapper diff --git a/configurations/errors.py b/configurations/errors.py new file mode 100644 index 0000000..3a59b51 --- /dev/null +++ b/configurations/errors.py @@ -0,0 +1,53 @@ +from typing import * +from functools import wraps +import sys +import os + + +class ValueRetrievalError(ValueError): + """ + Exception is raised when errors occur during the retrieval of a dynamic Value. + This can happen when the environment variable corresponding to the value is not defined. + """ + + def __init__(self, value_instance: 'configurations.values.Value', error_msg: str) -> None: + super().__init__(error_msg) + self.value_instance = value_instance + self.error_msg = error_msg + + +class ValueProcessingError(ValueError): + """ + Exception that is raised when a dynamic Value failed to be processed after retrieval. + Processing could be i.e. converting from string to a native datatype but also validation. + """ + + def __init__(self, value_instance: 'configurations.values.Value', str_value: str, error_msg: str) -> None: + super().__init__(error_msg) + self.value_instance = value_instance + self.str_value = str_value + self.error_msg = error_msg + + +class TermStyles: + BOLD = "\033[1m" + RED = "\033[91m" + END = "\033[0m" + + +def with_error_handler(callee: Callable) -> Callable: + """ + A decorator which is designed to wrap django entry points with an error handler so that django-configuration + originated errors can be caught and rendered to the user in a readable format. + """ + @wraps(callee) + def wrapper(*args, **kwargs): + try: + return callee(*args, **kwargs) + except (ValueRetrievalError, ValueProcessingError) as e: + msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) \ + if os.isatty(sys.stderr.fileno()) \ + else str(e) + print(msg, file=sys.stderr) + + return wrapper diff --git a/configurations/fastcgi.py b/configurations/fastcgi.py index 476bbaa..4eeb955 100644 --- a/configurations/fastcgi.py +++ b/configurations/fastcgi.py @@ -1,5 +1,5 @@ from . import importer -from .error_handling import with_error_handler +from .errors import with_error_handler importer.install() diff --git a/configurations/management.py b/configurations/management.py index b175c8b..1817a9a 100644 --- a/configurations/management.py +++ b/configurations/management.py @@ -1,5 +1,5 @@ from . import importer -from .error_handling import with_error_handler +from .errors import with_error_handler importer.install(check_options=True) diff --git a/configurations/values.py b/configurations/values.py index 6642c95..2051718 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -8,34 +8,10 @@ from django.core import validators from django.core.exceptions import ValidationError, ImproperlyConfigured from django.utils.module_loading import import_string +from .errors import ValueRetrievalError, ValueProcessingError from .utils import getargspec -class ValueRetrievalError(ValueError): - """ - Exception is raised when errors occur during the retrieval of a dynamic Value. - This can happen when the environment variable corresponding to the value is not defined. - """ - - def __init__(self, value_instance: 'Value', error_msg: str) -> None: - super().__init__(error_msg) - self.value_instance = value_instance - self.error_msg = error_msg - - -class ValueProcessingError(ValueError): - """ - Exception that is raised when a dynamic Value failed to be processed after retrieval. - Processing could be i.e. converting from string to a native datatype but also validation. - """ - - def __init__(self, value_instance: 'Value', str_value: str, error_msg: str) -> None: - super().__init__(error_msg) - self.value_instance = value_instance - self.str_value = str_value - self.error_msg = error_msg - - def setup_value(target, name, value): actual_value = value.setup(name) # overwriting the original Value class with the result diff --git a/configurations/wsgi.py b/configurations/wsgi.py index af5f21a..d99d611 100644 --- a/configurations/wsgi.py +++ b/configurations/wsgi.py @@ -1,5 +1,5 @@ from . import importer -from .error_handling import with_error_handler +from .errors import with_error_handler importer.install() diff --git a/tests/test_values.py b/tests/test_values.py index 6eb1ca4..b7cd131 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -16,7 +16,8 @@ from configurations.values import (Value, BooleanValue, IntegerValue, DatabaseURLValue, EmailURLValue, CacheURLValue, BackendsValue, SearchURLValue, PositiveIntegerValue, CastingMixin, - setup_value, ValueRetrievalError, ValueProcessingError) + setup_value) +from configurations.errors import ValueRetrievalError, ValueProcessingError @contextmanager From 7de261544178d44bbe521f1c709092f0fe3a6b99 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 10:13:49 +0100 Subject: [PATCH 06/15] transport additional information in error types --- configurations/errors.py | 113 ++++++++++++++++++++++++++++----------- configurations/values.py | 82 +++++++++++++++------------- 2 files changed, 128 insertions(+), 67 deletions(-) diff --git a/configurations/errors.py b/configurations/errors.py index 3a59b51..da8bbe8 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -3,36 +3,86 @@ from functools import wraps import sys import os - -class ValueRetrievalError(ValueError): - """ - Exception is raised when errors occur during the retrieval of a dynamic Value. - This can happen when the environment variable corresponding to the value is not defined. - """ - - def __init__(self, value_instance: 'configurations.values.Value', error_msg: str) -> None: - super().__init__(error_msg) - self.value_instance = value_instance - self.error_msg = error_msg - - -class ValueProcessingError(ValueError): - """ - Exception that is raised when a dynamic Value failed to be processed after retrieval. - Processing could be i.e. converting from string to a native datatype but also validation. - """ - - def __init__(self, value_instance: 'configurations.values.Value', str_value: str, error_msg: str) -> None: - super().__init__(error_msg) - self.value_instance = value_instance - self.str_value = str_value - self.error_msg = error_msg +if TYPE_CHECKING: + from .values import Value class TermStyles: - BOLD = "\033[1m" - RED = "\033[91m" - END = "\033[0m" + BOLD = "\033[1m" if os.isatty(sys.stderr.fileno()) else "" + RED = "\033[91m" if os.isatty(sys.stderr.fileno()) else "" + END = "\033[0m" if os.isatty(sys.stderr.fileno()) else "" + + +class ConfigurationError(ValueError): + """ + Base error class that is used to indicate that something went wrong during configuration. + + This error type (and subclasses) is caught and pretty-printed by django-configurations so that an end-user does not + see an unwieldy traceback but instead a helpful error message. + """ + + def __init__(self, main_error_msg: str, explanation_lines: List[str]) -> None: + """ + :param main_error_msg: Main message that describes the error. + This will be displayed before all *explanation_lines* and in the traceback (although tracebacks are normally + not rendered) + :param explanation_lines: Additional lines of explanations which further describe the error or give hints on + how to fix it. + """ + super().__init__(main_error_msg) + self.main_error_msg = main_error_msg + self.explanation_lines = explanation_lines + + +class ValueRetrievalError(ConfigurationError): + """ + Exception that is raised when errors occur during the retrieval of a Value by one of the `Value` classes. + This can happen when the environment variable corresponding to the value is not defined. + """ + + def __init__(self, value_instance: "Value", *extra_explanation_lines: str): + """ + :param value_instance: The `Value` instance which caused the generation of this error + :param extra_explanation_lines: Extra lines that will be appended to `ConfigurationError.explanation_lines` + in addition the ones automatically generated from the provided *value_instance*. + """ + explanation_lines = list(extra_explanation_lines) + if value_instance.destination_name is not None: + explanation_lines.append(f"{value_instance.destination_name} is taken from the environment variable " + f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") + + super().__init__( + f"Value of {value_instance.destination_name} could not be retrieved from environment", + explanation_lines + ) + + +class ValueProcessingError(ConfigurationError): + """ + Exception that is raised when a dynamic Value failed to be processed by one of the `Value` classes after retrieval. + + Processing could be i.e. converting from string to a native datatype or validation. + """ + + def __init__(self, value_instance: "Value", raw_value: str, *extra_explanation_lines: str): + """ + :param value_instance: The `Value` instance which caused the generation of this error + :param raw_value: The raw value that was retrieved from the environment and which could not be processed further + :param extra_explanation_lines: Extra lines that will be prepended to `ConfigurationError.explanation_lines` + in addition the ones automatically generated from the provided *value_instance*. + """ + error = f"{value_instance.destination_name} was given an invalid value" + if hasattr(value_instance, "message"): + error += ": " + value_instance.message.format(raw_value) + + explanation_lines = list(extra_explanation_lines) + if value_instance.destination_name is not None: + explanation_lines.append(f"{value_instance.destination_name} is taken from the environment variable " + f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") + + explanation_lines.append(f"'{raw_value}' was received but that is invalid") + + super().__init__(error, explanation_lines) def with_error_handler(callee: Callable) -> Callable: @@ -40,14 +90,15 @@ def with_error_handler(callee: Callable) -> Callable: A decorator which is designed to wrap django entry points with an error handler so that django-configuration originated errors can be caught and rendered to the user in a readable format. """ + @wraps(callee) def wrapper(*args, **kwargs): try: return callee(*args, **kwargs) - except (ValueRetrievalError, ValueProcessingError) as e: - msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) \ - if os.isatty(sys.stderr.fileno()) \ - else str(e) + except ConfigurationError as e: + msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) + for line in e.explanation_lines: + msg += f"\n {line}" print(msg, file=sys.stderr) return wrapper diff --git a/configurations/values.py b/configurations/values.py index 2051718..8829d43 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -2,13 +2,12 @@ import ast import copy import decimal import os -import sys from django.core import validators from django.core.exceptions import ValidationError, ImproperlyConfigured from django.utils.module_loading import import_string -from .errors import ValueRetrievalError, ValueProcessingError +from .errors import ValueRetrievalError, ValueProcessingError, ConfigurationError from .utils import getargspec @@ -71,6 +70,7 @@ class Value: self.environ_prefix = environ_prefix self.environ_name = environ_name self.environ_required = environ_required + self.destination_name = None def __str__(self): return str(self.value) @@ -87,33 +87,46 @@ class Value: # Compatibility with python 2 __nonzero__ = __bool__ - def full_environ_name(self, name): + @property + def full_environ_name(self): + """ + The full name of the environment variable (including prefix and capitalization) from which this value should be + retrieved + """ if self.environ_name: environ_name = self.environ_name else: - environ_name = name.upper() + environ_name = self.destination_name.upper() if self.environ_prefix: environ_name = '{0}_{1}'.format(self.environ_prefix, environ_name) return environ_name def setup(self, name): + """ + Set up this value instance by retrieving the configured value from the environment and converting it to a native + python data type + + :param name: Destination name for which this value is used. + For example in the scenario of `DEBUG = Value()` in a `Configuration` subclass, this would be `DEBUG`. + """ + self.destination_name = name value = self.default if self.environ: - full_environ_name = self.full_environ_name(name) + full_environ_name = self.full_environ_name if full_environ_name in os.environ: value = self.to_python(os.environ[full_environ_name]) elif self.environ_required: - raise ValueRetrievalError(self, 'Value {0!r} is required to be set as the ' - 'environment variable {1!r}' - .format(name, full_environ_name)) + raise ValueRetrievalError(self) self.value = value return value - def to_python(self, value): + def to_python(self, value: str): """ - Convert the given value of a environment variable into an + Convert the given value of an environment variable into an appropriate Python representation of the value. - This should be overriden when subclassing. + This should be overridden when subclassing. + + :param value: The value that should be converted to a python representation """ return value @@ -132,15 +145,14 @@ class BooleanValue(Value): raise ImproperlyConfigured('Default value {0!r} is not a ' 'boolean value'.format(self.default)) - def to_python(self, value): + def to_python(self, value: str): normalized_value = value.strip().lower() if normalized_value in self.true_values: return True elif normalized_value in self.false_values: return False else: - raise ValueProcessingError(self, value, 'Cannot interpret ' - 'boolean value {0!r}'.format(value)) + raise ValueProcessingError(self, value) class CastingMixin: @@ -167,14 +179,14 @@ class CastingMixin: except TypeError: self._params = {} - def to_python(self, value): + def to_python(self, value: str): try: if self._params: return self._caster(value, **self._params) else: return self._caster(value) except self.exception: - raise ValueProcessingError(self, value, self.message.format(value)) + raise ValueProcessingError(self, value) class IntegerValue(CastingMixin, Value): @@ -183,10 +195,10 @@ class IntegerValue(CastingMixin, Value): class PositiveIntegerValue(IntegerValue): - def to_python(self, value): + def to_python(self, value: str): int_value = super().to_python(value) if int_value < 0: - raise ValueProcessingError(self, value, self.message.format(value)) + raise ValueProcessingError(self, value, f"Value needs to be positive or zero but {int_value} isn't") return int_value @@ -210,7 +222,7 @@ class SequenceValue(Value): converter = None def __init__(self, *args, **kwargs): - msg = 'Cannot interpret {0} item {{0!r}} in {0} {{1!r}}' + msg = 'Cannot interpret {0} item in {0} {{0!r}}' self.message = msg.format(self.sequence_type.__name__) self.separator = kwargs.pop('separator', ',') converter = kwargs.pop('converter', None) @@ -232,10 +244,10 @@ class SequenceValue(Value): try: converted_values.append(self.converter(value)) except (TypeError, ValueError): - raise ValueProcessingError(self, self.separator.join(sequence), self.message.format(value, value)) + raise ValueProcessingError(self, self.separator.join(sequence)) return self.sequence_type(converted_values) - def to_python(self, value): + def to_python(self, value: str): split_value = [v.strip() for v in value.strip().split(self.separator)] # removing empty items value_list = self.sequence_type(filter(None, split_value)) @@ -271,7 +283,7 @@ class SingleNestedSequenceValue(SequenceValue): return self.sequence_type(converted_sequences) return self.sequence_type(super()._convert(items)) - def to_python(self, value): + def to_python(self, value: str): split_value = [ v.strip() for v in value.strip().split(self.seq_separator) ] @@ -297,13 +309,11 @@ class BackendsValue(ListValue): try: import_string(value) except ImportError as err: - raise ValueProcessingError(self, value, str(err)).with_traceback(sys.exc_info()[2]) + raise ValueProcessingError(self, value) return value class SetValue(ListValue): - message = 'Cannot interpret set item {0!r} in set {1!r}' - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if self.default is None: @@ -311,7 +321,7 @@ class SetValue(ListValue): else: self.default = set(self.default) - def to_python(self, value): + def to_python(self, value: str): return set(super().to_python(value)) @@ -325,16 +335,16 @@ class DictValue(Value): else: self.default = dict(self.default) - def to_python(self, value): + def to_python(self, value: str): value = super().to_python(value) if not value: return {} try: evaled_value = ast.literal_eval(value) except ValueError: - raise ValueProcessingError(self, value, self.message.format(value)) + raise ValueProcessingError(self, value) if not isinstance(evaled_value, dict): - raise ValueProcessingError(self, value, self.message.format(value)) + raise ValueProcessingError(self, value) return evaled_value @@ -357,13 +367,13 @@ class ValidationMixin: try: self.to_python(self.default) except ValueProcessingError as e: - raise ImproperlyConfigured(e.error_msg) + raise ImproperlyConfigured(e.main_error_msg) from e - def to_python(self, value): + def to_python(self, value: str): try: self._validator(value) - except ValidationError: - raise ValueProcessingError(self, value, self.message.format(value)) + except ValidationError as e: + raise ValueProcessingError(self, value, f"Validation failed: {e.message}") else: return value @@ -401,7 +411,7 @@ class PathValue(Value): value = super().setup(name) value = os.path.expanduser(value) if self.check_exists and not os.path.exists(value): - raise ValueProcessingError(self, value, 'Path {0!r} does not exist.'.format(value)) + raise ValueProcessingError(self, value, f"Path {value} does not exist") return os.path.abspath(value) @@ -418,7 +428,7 @@ class SecretValue(Value): def setup(self, name): value = super().setup(name) if not value: - raise ValueRetrievalError(self, 'Secret value {0!r} is not set'.format(name)) + raise ValueRetrievalError(self) return value @@ -452,7 +462,7 @@ class DictBackendMixin(Value): else: self.default = self.to_python(self.default) - def to_python(self, value): + def to_python(self, value: str): value = super().to_python(value) return {self.alias: value} From a3b720f31a65fd1172133689077cdbb0aab04bfe Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 10:42:06 +0100 Subject: [PATCH 07/15] implement error accumulation This way, multiple ConfigurationErrors are caught during setup, accumulated and printed all at once --- configurations/base.py | 10 +++++++++- configurations/errors.py | 25 +++++++++++++++++++++---- configurations/importer.py | 13 +++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/configurations/base.py b/configurations/base.py index 14185fe..d71cf89 100644 --- a/configurations/base.py +++ b/configurations/base.py @@ -4,6 +4,7 @@ import re from django.conf import global_settings from django.core.exceptions import ImproperlyConfigured +from .errors import ConfigurationError, SetupError from .utils import uppercase_attributes from .values import Value, setup_value @@ -142,6 +143,13 @@ class Configuration(metaclass=ConfigurationBase): @classmethod def setup(cls): + exceptions = [] for name, value in uppercase_attributes(cls).items(): if isinstance(value, Value): - setup_value(cls, name, value) + try: + setup_value(cls, name, value) + except ConfigurationError as err: + exceptions.append(err) + + if len(exceptions) > 0: + raise SetupError(f"Couldn't setup values of configuration {cls.__name__}", exceptions) diff --git a/configurations/errors.py b/configurations/errors.py index da8bbe8..102badf 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -13,6 +13,20 @@ class TermStyles: END = "\033[0m" if os.isatty(sys.stderr.fileno()) else "" +class SetupError(Exception): + """ + Exception that gets raised when a configuration class cannot be set up by the importer + """ + def __init__(self, msg: str, child_errors: List['ConfigurationError'] = None) -> None: + """ + :param step_verb: Which step the importer tried to perform (e.g. import, setup) + :param configuration_path: The full module path of the configuration that was supposed to be set up + :param child_errors: Optional child configuration errors that caused this error + """ + super().__init__(msg) + self.child_errors = child_errors or [] + + class ConfigurationError(ValueError): """ Base error class that is used to indicate that something went wrong during configuration. @@ -95,10 +109,13 @@ def with_error_handler(callee: Callable) -> Callable: def wrapper(*args, **kwargs): try: return callee(*args, **kwargs) - except ConfigurationError as e: - msg = "{}{}{}".format(TermStyles.RED + TermStyles.BOLD, e, TermStyles.END) - for line in e.explanation_lines: - msg += f"\n {line}" + except SetupError as e: + msg = f"{str(e)}" + for child_error in e.child_errors: + msg += f"\n * {child_error.main_error_msg}" + for explanation_line in child_error.explanation_lines: + msg += f"\n - {explanation_line}" + print(msg, file=sys.stderr) return wrapper diff --git a/configurations/importer.py b/configurations/importer.py index e3573f4..df5f6b9 100644 --- a/configurations/importer.py +++ b/configurations/importer.py @@ -8,6 +8,7 @@ from django.conf import ENVIRONMENT_VARIABLE as SETTINGS_ENVIRONMENT_VARIABLE from django.core.exceptions import ImproperlyConfigured from django.core.management import base +from .errors import SetupError, ConfigurationError from .utils import uppercase_attributes, reraise from .values import Value, setup_value @@ -149,10 +150,10 @@ class ConfigurationLoader: try: cls = getattr(mod, self.name) - except AttributeError as err: # pragma: no cover - reraise(err, "Couldn't find configuration '{0}' " - "in module '{1}'".format(self.name, - mod.__package__)) + except AttributeError: # pragma: no cover + raise SetupError(f"Couldn't find configuration '{self.name}' in module {mod.__package__}.\n" + f"Hint: '{self.name}' is taken from the environment variable '{CONFIGURATION_ENVIRONMENT_VARIABLE}'" + f"and '{mod.__package__}' from the environment variable '{SETTINGS_ENVIRONMENT_VARIABLE}'.") try: cls.pre_setup() cls.setup() @@ -172,6 +173,10 @@ class ConfigurationLoader: self.name)) cls.post_setup() + except SetupError: + raise + except ConfigurationError as err: + raise SetupError(f"Couldn't setup configuration '{cls_path}'", [err]) except Exception as err: reraise(err, "Couldn't setup configuration '{0}'".format(cls_path)) From 50ac28b667ddabd6060ba2cdadc6eba9eefa6202 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 11:03:13 +0100 Subject: [PATCH 08/15] add support for adding help_text to values Now Value classes accept a 'help_text' keyword which will get printed when a ValueRetrievalError or ValueProcessingError occurs --- configurations/errors.py | 27 ++++++++++++++++----------- configurations/values.py | 3 ++- test_project/test_project/settings.py | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/configurations/errors.py b/configurations/errors.py index 102badf..9804896 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -13,10 +13,24 @@ class TermStyles: END = "\033[0m" if os.isatty(sys.stderr.fileno()) else "" +def extract_explanation_lines_from_value(value_instance: 'Value') -> List[str]: + result = [] + + if value_instance.help_text is not None: + result.append(f"Help: {value_instance.help_text}") + + if value_instance.destination_name is not None: + result.append(f"{value_instance.destination_name} is taken from the environment variable " + f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") + + return result + + class SetupError(Exception): """ Exception that gets raised when a configuration class cannot be set up by the importer """ + def __init__(self, msg: str, child_errors: List['ConfigurationError'] = None) -> None: """ :param step_verb: Which step the importer tried to perform (e.g. import, setup) @@ -60,14 +74,9 @@ class ValueRetrievalError(ConfigurationError): :param extra_explanation_lines: Extra lines that will be appended to `ConfigurationError.explanation_lines` in addition the ones automatically generated from the provided *value_instance*. """ - explanation_lines = list(extra_explanation_lines) - if value_instance.destination_name is not None: - explanation_lines.append(f"{value_instance.destination_name} is taken from the environment variable " - f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") - super().__init__( f"Value of {value_instance.destination_name} could not be retrieved from environment", - explanation_lines + list(extra_explanation_lines) + extract_explanation_lines_from_value(value_instance) ) @@ -89,11 +98,7 @@ class ValueProcessingError(ConfigurationError): if hasattr(value_instance, "message"): error += ": " + value_instance.message.format(raw_value) - explanation_lines = list(extra_explanation_lines) - if value_instance.destination_name is not None: - explanation_lines.append(f"{value_instance.destination_name} is taken from the environment variable " - f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") - + explanation_lines = list(extra_explanation_lines) + extract_explanation_lines_from_value(value_instance) explanation_lines.append(f"'{raw_value}' was received but that is invalid") super().__init__(error, explanation_lines) diff --git a/configurations/values.py b/configurations/values.py index 8829d43..a0ddce6 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -58,7 +58,7 @@ class Value: return instance def __init__(self, default=None, environ=True, environ_name=None, - environ_prefix='DJANGO', environ_required=False, + environ_prefix='DJANGO', environ_required=False, help_text=None, *args, **kwargs): if isinstance(default, Value) and default.default is not None: self.default = copy.copy(default.default) @@ -71,6 +71,7 @@ class Value: self.environ_name = environ_name self.environ_required = environ_required self.destination_name = None + self.help_text = help_text def __str__(self): return str(self.value) diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index 879a3b8..0cf7a74 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -4,7 +4,7 @@ from configurations import Configuration, values class Base(Configuration): # Django settings for test_project project. - DEBUG = values.BooleanValue(True, environ=True) + DEBUG = values.BooleanValue(True, environ=True, help_text="Enables or disables django debug mode") ADMINS = ( # ('Your Name', 'your_email@example.com'), From 12ffda4760f8b09101708e7a861773ee57960f1e Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 11:05:20 +0100 Subject: [PATCH 09/15] add support for adding reference links to values Now Value classes accept a 'help_reference' keyword which will get printed when a ValueRetrievalError or ValueProcessingError occurs --- configurations/errors.py | 3 +++ configurations/values.py | 3 ++- test_project/test_project/settings.py | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configurations/errors.py b/configurations/errors.py index 9804896..7128317 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -19,6 +19,9 @@ def extract_explanation_lines_from_value(value_instance: 'Value') -> List[str]: if value_instance.help_text is not None: result.append(f"Help: {value_instance.help_text}") + if value_instance.help_reference is not None: + result.append(f"Reference: {value_instance.help_reference}") + if value_instance.destination_name is not None: result.append(f"{value_instance.destination_name} is taken from the environment variable " f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") diff --git a/configurations/values.py b/configurations/values.py index a0ddce6..b44f809 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -59,7 +59,7 @@ class Value: def __init__(self, default=None, environ=True, environ_name=None, environ_prefix='DJANGO', environ_required=False, help_text=None, - *args, **kwargs): + help_reference=None, *args, **kwargs): if isinstance(default, Value) and default.default is not None: self.default = copy.copy(default.default) else: @@ -72,6 +72,7 @@ class Value: self.environ_required = environ_required self.destination_name = None self.help_text = help_text + self.help_reference = help_reference def __str__(self): return str(self.value) diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py index 0cf7a74..efa32ab 100644 --- a/test_project/test_project/settings.py +++ b/test_project/test_project/settings.py @@ -4,7 +4,8 @@ from configurations import Configuration, values class Base(Configuration): # Django settings for test_project project. - DEBUG = values.BooleanValue(True, environ=True, help_text="Enables or disables django debug mode") + DEBUG = values.BooleanValue(True, environ=True, help_text="Enables or disables django debug mode", + help_reference="https://docs.djangoproject.com/en/dev/ref/settings/#debug") ADMINS = ( # ('Your Name', 'your_email@example.com'), From 42ab54fc4d2cfd668b223ae31fe0eaba737e6389 Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 11:14:15 +0100 Subject: [PATCH 10/15] fix failing 'test_email_url_value' test See https://github.com/jazzband/django-configurations/issues/332 --- tests/test_values.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_values.py b/tests/test_values.py index b7cd131..a1d843b 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -425,6 +425,7 @@ class ValueTests(TestCase): 'EMAIL_HOST_PASSWORD': 'password', 'EMAIL_HOST_USER': 'user@domain.com', 'EMAIL_PORT': 587, + 'EMAIL_TIMEOUT': None, 'EMAIL_USE_SSL': False, 'EMAIL_USE_TLS': True}) with env(EMAIL_URL='console://'): @@ -435,6 +436,7 @@ class ValueTests(TestCase): 'EMAIL_HOST_PASSWORD': None, 'EMAIL_HOST_USER': None, 'EMAIL_PORT': None, + 'EMAIL_TIMEOUT': None, 'EMAIL_USE_SSL': False, 'EMAIL_USE_TLS': False}) with env(EMAIL_URL='smtps://user@domain.com:password@smtp.example.com:wrong'): # noqa: E501 From a7f06061de041c38b25f243ff969341dc4c14a6f Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 11:19:13 +0100 Subject: [PATCH 11/15] fix failing 'test_cache_url_value' test The test failed for two reasons. One was me changing the error string in a previous commit which I have adapted the test for now. The other was that django-cache-url changed its behavior. See https://github.com/jazzband/django-configurations/issues/332 --- tests/test_values.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_values.py b/tests/test_values.py index a1d843b..e3e68d2 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -2,6 +2,7 @@ import decimal import os from contextlib import contextmanager +from django import VERSION as DJANGO_VERSION from django.test import TestCase from django.core.exceptions import ImproperlyConfigured @@ -61,14 +62,14 @@ class ValueTests(TestCase): def test_environ_required(self): for ValueClass in (Value, BooleanValue, IntegerValue, - FloatValue, DecimalValue, ListValue, - TupleValue, SingleNestedTupleValue, - SingleNestedListValue, SetValue, - DictValue, URLValue, EmailValue, IPValue, - RegexValue, PathValue, SecretValue, - DatabaseURLValue, EmailURLValue, - CacheURLValue, BackendsValue, - SearchURLValue, PositiveIntegerValue): + FloatValue, DecimalValue, ListValue, + TupleValue, SingleNestedTupleValue, + SingleNestedListValue, SetValue, + DictValue, URLValue, EmailValue, IPValue, + RegexValue, PathValue, SecretValue, + DatabaseURLValue, EmailURLValue, + CacheURLValue, BackendsValue, + SearchURLValue, PositiveIntegerValue): value = ValueClass(environ_required=True) self.assertRaises(ValueRetrievalError, value.setup, "TEST") @@ -445,7 +446,7 @@ class ValueTests(TestCase): def test_cache_url_value(self): cache_setting = { 'default': { - 'BACKEND': 'django_redis.cache.RedisCache', + 'BACKEND': 'django_redis.cache.RedisCache' if DJANGO_VERSION[0] < 4 else 'django.core.cache.backends.redis.RedisCache', 'LOCATION': 'redis://host:6379/1', } } @@ -465,7 +466,7 @@ class ValueTests(TestCase): value.setup('TEST') self.assertEqual( cm.exception.args[0], - "Cannot interpret cache URL value 'redis://user@host:port/1'") + "TEST was given an invalid value: Cannot interpret cache URL value 'redis://user@host:port/1'") def test_search_url_value(self): value = SearchURLValue() From fd1b5715946c9cf0f02002bc1ac71502937502cf Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 13:37:18 +0100 Subject: [PATCH 12/15] fix flake8 warnings --- configurations/errors.py | 2 +- configurations/values.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configurations/errors.py b/configurations/errors.py index 7128317..cfca60e 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -1,4 +1,4 @@ -from typing import * +from typing import TYPE_CHECKING, List, Callable from functools import wraps import sys import os diff --git a/configurations/values.py b/configurations/values.py index b44f809..73d2046 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -7,7 +7,7 @@ from django.core import validators from django.core.exceptions import ValidationError, ImproperlyConfigured from django.utils.module_loading import import_string -from .errors import ValueRetrievalError, ValueProcessingError, ConfigurationError +from .errors import ValueRetrievalError, ValueProcessingError from .utils import getargspec @@ -311,7 +311,7 @@ class BackendsValue(ListValue): try: import_string(value) except ImportError as err: - raise ValueProcessingError(self, value) + raise ValueProcessingError(self, value) from err return value From 5cd3a8610698d596391046bf3d41e0834638310b Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 14:25:28 +0100 Subject: [PATCH 13/15] implement example generators Example generators are an addition to the 'Value' classes that allow the error handler to display example values for the failed value. For example if no django secret key is found to be defined in the environment, an example (secure) value for such a secret key can now be generated on the fly and be suggested to the user --- configurations/errors.py | 3 ++ configurations/example_generators.py | 47 ++++++++++++++++++++++++++++ configurations/values.py | 5 +-- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 configurations/example_generators.py diff --git a/configurations/errors.py b/configurations/errors.py index cfca60e..1f4a984 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -26,6 +26,9 @@ def extract_explanation_lines_from_value(value_instance: 'Value') -> List[str]: result.append(f"{value_instance.destination_name} is taken from the environment variable " f"{value_instance.full_environ_name} as a {type(value_instance).__name__}") + if value_instance.example_generator is not None: + result.append(f"Example value: '{value_instance.example_generator()}'") + return result diff --git a/configurations/example_generators.py b/configurations/example_generators.py new file mode 100644 index 0000000..e52983c --- /dev/null +++ b/configurations/example_generators.py @@ -0,0 +1,47 @@ +from typing import Callable +import secrets +import base64 +from django.core.management.utils import get_random_secret_key +from django.utils.crypto import get_random_string, RANDOM_STRING_CHARS + + +def gen_django_secret_key() -> str: + """ + Generate a cryptographically secure random string that can safely be used as a SECRET_KEY in django + """ + return get_random_secret_key() + + +def gen_random_string(length: int, allowed_chars: str = RANDOM_STRING_CHARS) -> Callable[[], str]: + """ + Create a parameterized generator which generates a cryptographically secure random string of the given length + containing the given characters. + """ + def generate() -> str: + return get_random_string(length, allowed_chars) + return generate + + +def gen_bytes(length: int, encoding: str) -> Callable[[], str]: + """ + Create a parameterized generator which generates a cryptographically secure random assortments of bytes of the given + length and encoded in the given format + + :param length: How many bytes should be generated. Not how long the encoded string will be. + :param encoding: How the generated bytes should be encoded. + Accepted values are "base64", "base64_urlsafe" and "hex" (case is ignored) + """ + encoding = encoding.lower() + if encoding not in ("base64", "base64_urlsafe", "hex"): + raise ValueError(f"Cannot gen_bytes with encoding '{encoding}'. Valid encodings are 'base64', 'base64_urlsafe'" + f" and 'hex'") + + def generate() -> str: + b = secrets.token_bytes(length) + if encoding == "base64": + return base64.standard_b64encode(b).decode("ASCII") + elif encoding == "base64_urlsafe": + return base64.urlsafe_b64encode(b).decode("ASCII") + elif encoding == "hex": + return b.hex().upper() + return generate diff --git a/configurations/values.py b/configurations/values.py index 73d2046..98d3c26 100644 --- a/configurations/values.py +++ b/configurations/values.py @@ -58,8 +58,8 @@ class Value: return instance def __init__(self, default=None, environ=True, environ_name=None, - environ_prefix='DJANGO', environ_required=False, help_text=None, - help_reference=None, *args, **kwargs): + environ_prefix='DJANGO', environ_required=False, example_generator=None, + help_text=None, help_reference=None, *args, **kwargs): if isinstance(default, Value) and default.default is not None: self.default = copy.copy(default.default) else: @@ -73,6 +73,7 @@ class Value: self.destination_name = None self.help_text = help_text self.help_reference = help_reference + self.example_generator = example_generator def __str__(self): return str(self.value) From 3b96d16510f56b5f726cfa105f4de664720a27ab Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 14:37:22 +0100 Subject: [PATCH 14/15] add tests for example generators --- configurations/example_generators.py | 22 +++++++++++++---- tests/test_example_generators.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 tests/test_example_generators.py diff --git a/configurations/example_generators.py b/configurations/example_generators.py index e52983c..2da7d09 100644 --- a/configurations/example_generators.py +++ b/configurations/example_generators.py @@ -1,8 +1,17 @@ from typing import Callable import secrets import base64 + +import django from django.core.management.utils import get_random_secret_key -from django.utils.crypto import get_random_string, RANDOM_STRING_CHARS +from django.utils.crypto import get_random_string + +if django.VERSION[0] > 3 or \ + (django.VERSION[0] == 3 and django.VERSION[1] >= 2): + # RANDOM_STRING_CHARS was only introduced in django 3.2 + from django.utils.crypto import RANDOM_STRING_CHARS +else: + RANDOM_STRING_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" # pragma: no cover def gen_django_secret_key() -> str: @@ -17,9 +26,11 @@ def gen_random_string(length: int, allowed_chars: str = RANDOM_STRING_CHARS) -> Create a parameterized generator which generates a cryptographically secure random string of the given length containing the given characters. """ - def generate() -> str: + + def _gen_random_string() -> str: return get_random_string(length, allowed_chars) - return generate + + return _gen_random_string def gen_bytes(length: int, encoding: str) -> Callable[[], str]: @@ -36,7 +47,7 @@ def gen_bytes(length: int, encoding: str) -> Callable[[], str]: raise ValueError(f"Cannot gen_bytes with encoding '{encoding}'. Valid encodings are 'base64', 'base64_urlsafe'" f" and 'hex'") - def generate() -> str: + def _gen_bytes() -> str: b = secrets.token_bytes(length) if encoding == "base64": return base64.standard_b64encode(b).decode("ASCII") @@ -44,4 +55,5 @@ def gen_bytes(length: int, encoding: str) -> Callable[[], str]: return base64.urlsafe_b64encode(b).decode("ASCII") elif encoding == "hex": return b.hex().upper() - return generate + + return _gen_bytes diff --git a/tests/test_example_generators.py b/tests/test_example_generators.py new file mode 100644 index 0000000..8f8bda5 --- /dev/null +++ b/tests/test_example_generators.py @@ -0,0 +1,35 @@ +import base64 +from django.test import TestCase +from configurations.example_generators import gen_bytes, gen_random_string, gen_django_secret_key + + +class ExampleGeneratorsTestCase(TestCase): + def test_generators_dont_raise_exceptions(self): + for gen in [gen_bytes(64, "hex"), gen_bytes(64, "base64"), gen_bytes(64, "base64_urlsafe"), + gen_random_string(16, "ab"), gen_random_string(5), + gen_django_secret_key]: + with self.subTest(gen.__name__): + gen() + + # gen_django_secret_key() and gen_random_string() are not tested beyond the above general test case + # because they are just wrappers around existing django utilities. + # They are thus assumed to work. + + def test_gen_bytes(self): + with self.subTest("base64"): + result = gen_bytes(64, "base64")() + b = base64.standard_b64decode(result.encode("ASCII")) + self.assertEqual(len(b), 64) + + with self.subTest("base64_urlsafe"): + result = gen_bytes(64, "base64_urlsafe")() + b = base64.urlsafe_b64decode(result.encode("ASCII")) + self.assertEqual(len(b), 64) + + with self.subTest("hex"): + result = gen_bytes(64, "hex")() + b = bytes.fromhex(result) + self.assertEqual(len(b), 64) + + with self.subTest("invalid"): + self.assertRaises(ValueError, gen_bytes, 64, "invalid") From b21f207b4f2bdf170d08c5d5fda6c77fa9b20abc Mon Sep 17 00:00:00 2001 From: Finn-Thorben Sell Date: Thu, 24 Mar 2022 17:07:16 +0100 Subject: [PATCH 15/15] add tests for error handling --- configurations/errors.py | 2 +- tests/test_error_handling.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/test_error_handling.py diff --git a/configurations/errors.py b/configurations/errors.py index 1f4a984..b339801 100644 --- a/configurations/errors.py +++ b/configurations/errors.py @@ -4,7 +4,7 @@ import sys import os if TYPE_CHECKING: - from .values import Value + from .values import Value # pragma: no cover class TermStyles: diff --git a/tests/test_error_handling.py b/tests/test_error_handling.py new file mode 100644 index 0000000..7f5d0ff --- /dev/null +++ b/tests/test_error_handling.py @@ -0,0 +1,43 @@ +import io +from unittest.mock import patch + +from django.test import TestCase + +from configurations.errors import ValueRetrievalError, SetupError, with_error_handler +from configurations.values import Value + + +class ErrorHandlingTestCase(TestCase): + def test_help_text_in_explanation_lines(self): + value_instance = Value(help_text="THIS IS A TEST") + exception = ValueRetrievalError(value_instance) + self.assertIn("Help: THIS IS A TEST", exception.explanation_lines) + + def test_help_reference_in_explanation_lines(self): + value_instance = Value(help_reference="https://example.com") + exception = ValueRetrievalError(value_instance) + self.assertIn("Reference: https://example.com", exception.explanation_lines) + + def test_example_in_explanation_lines(self): + value_instance = Value(example_generator=lambda: "test") + exception = ValueRetrievalError(value_instance) + self.assertIn("Example value: 'test'", exception.explanation_lines) + + def test_error_handler_rendering(self): + # setup + with patch("configurations.errors.sys.stderr", new=io.StringIO()) as mock: + def inner(): + try: + value_instance = Value(environ_required=True) + value_instance.setup("TEST") + except ValueRetrievalError as err: + raise SetupError("This is a test exception", [err]) + + # execution + with_error_handler(inner)() + + # verification + self.assertEqual(mock.getvalue().strip(), + "This is a test exception\n" + " * Value of TEST could not be retrieved from environment\n" + " - TEST is taken from the environment variable DJANGO_TEST as a Value")