From c61bfb070c4704eb2e3049c0eb153bbff20e5c0e Mon Sep 17 00:00:00 2001 From: Vasiliy Polyakov Date: Mon, 24 Apr 2023 17:56:16 +0500 Subject: [PATCH] Add variable expansion (fix #421) - Expand variables referenced as `$VAR` or `${VAR}`. - Detect infinite recursion in expansion (self-reference). --- CHANGELOG.rst | 2 ++ docs/quickstart.rst | 22 ++++++++++++++ environ/environ.py | 66 ++++++++++++++++++++++++++++++++-------- tests/test_expansion.py | 27 ++++++++++++++++ tests/test_expansion.txt | 9 ++++++ 5 files changed, 114 insertions(+), 12 deletions(-) create mode 100755 tests/test_expansion.py create mode 100755 tests/test_expansion.txt diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 682b1072..b8daeff1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ Added +++++ - Added support for secure Elasticsearch connections `#463 `_. +- Added variable expansion + `#468 `_. Changed +++++++ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 62473838..2ab100fd 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -23,6 +23,28 @@ And use it with ``settings.py`` as follows: :start-after: -code-begin- :end-before: -overview- +Variables can contain references to another variables: ``$VAR`` or ``${VAR}``. +Referenced variables are searched in the environment and within all definitions +in the ``.env`` file. References are checked for recursion (self-reference). +Exception is thrown if any reference results in infinite loop on any level +of recursion. Variable values are substituted similar to shell parameter +expansion. Example: + +.. code-block:: shell + + # shell + export POSTGRES_USERNAME='user' POSTGRES_PASSWORD='SECRET' + +.. code-block:: shell + + # .env + POSTGRES_HOSTNAME='example.com' + POSTGRES_DB='database' + DATABASE_URL="postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOSTNAME}:5432/${POSTGRES_DB}" + +The value of ``DATABASE_URL`` variable will become +``postgres://user:SECRET@example.com:5432/database``. + The ``.env`` file should be specific to the environment and not checked into version control, it is best practice documenting the ``.env`` file with an example. For example, you can also add ``.env.dist`` with a template of your variables to diff --git a/environ/environ.py b/environ/environ.py index 8d09258e..f045639e 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -17,7 +17,9 @@ import os import re import sys +import threading import warnings +from os.path import expandvars from urllib.parse import ( parse_qs, ParseResult, @@ -40,6 +42,9 @@ Openable = (str, os.PathLike) logger = logging.getLogger(__name__) +# Variables which values should not be expanded +NOT_EXPANDED = 'DJANGO_SECRET_KEY', 'CACHE_URL' + def _cast(value): # Safely evaluate an expression node or a string containing a Python @@ -189,7 +194,11 @@ class Env: for s in ('', 's')] CLOUDSQL = 'cloudsql' + VAR = re.compile(r'(?[A-Z_][0-9A-Z_]*)}?', + re.IGNORECASE) + def __init__(self, **scheme): + self._local = threading.local() self.smart_cast = True self.escape_proxy = False self.prefix = "" @@ -343,9 +352,13 @@ def path(self, var, default=NOTSET, **kwargs): """ return Path(self.get_value(var, default=default), **kwargs) - def get_value(self, var, cast=None, default=NOTSET, parse_default=False): + def get_value(self, var, cast=None, # pylint: disable=R0913 + default=NOTSET, parse_default=False, add_prefix=True): """Return value for given environment variable. + - Expand variables referenced as ``$VAR`` or ``${VAR}``. + - Detect infinite recursion in expansion (self-reference). + :param str var: Name of variable. :param collections.abc.Callable or None cast: @@ -354,15 +367,33 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): If var not present in environ, return this instead. :param bool parse_default: Force to parse default. + :param bool add_prefix: + Whether to add prefix to variable name. :returns: Value from environment or default (if set). :rtype: typing.IO[typing.Any] """ - + var_name = f'{self.prefix}{var}' if add_prefix else var + if not hasattr(self._local, 'vars'): + self._local.vars = set() + if var_name in self._local.vars: + error_msg = f"Environment variable '{var_name}' recursively "\ + "references itself (eventually)" + raise ImproperlyConfigured(error_msg) + + self._local.vars.add(var_name) + try: + return self._get_value( + var_name, cast=cast, default=default, + parse_default=parse_default) + finally: + self._local.vars.remove(var_name) + + def _get_value(self, var_name, cast=None, default=NOTSET, + parse_default=False): logger.debug( "get '%s' casted as '%s' with default '%s'", - var, cast, default) + var_name, cast, default) - var_name = f'{self.prefix}{var}' if var_name in self.scheme: var_info = self.scheme[var_name] @@ -388,26 +419,37 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False): value = self.ENVIRON[var_name] except KeyError as exc: if default is self.NOTSET: - error_msg = f'Set the {var} environment variable' + error_msg = f'Set the {var_name} environment variable' raise ImproperlyConfigured(error_msg) from exc value = default + # Expand variables + if isinstance(value, (bytes, str)) and var_name not in NOT_EXPANDED: + def repl(match_): + return self.get_value( + match_.group('name'), cast=cast, default=default, + parse_default=parse_default, add_prefix=False) + + is_bytes = isinstance(value, bytes) + if is_bytes: + value = value.decode('utf-8') + value = self.VAR.sub(repl, value) + value = expandvars(value) + if is_bytes: + value = value.encode('utf-8') + # Resolve any proxied values prefix = b'$' if isinstance(value, bytes) else '$' escape = rb'\$' if isinstance(value, bytes) else r'\$' - if hasattr(value, 'startswith') and value.startswith(prefix): - value = value.lstrip(prefix) - value = self.get_value(value, cast=cast, default=default) if self.escape_proxy and hasattr(value, 'replace'): value = value.replace(escape, prefix) # Smart casting - if self.smart_cast: - if cast is None and default is not None and \ - not isinstance(default, NoValue): - cast = type(default) + if self.smart_cast and cast is None and default is not None \ + and not isinstance(default, NoValue): + cast = type(default) value = None if default is None and value == '' else value diff --git a/tests/test_expansion.py b/tests/test_expansion.py new file mode 100755 index 00000000..757ee9a2 --- /dev/null +++ b/tests/test_expansion.py @@ -0,0 +1,27 @@ +import pytest + +from environ import Env, Path +from environ.compat import ImproperlyConfigured + + +class TestExpansion: + def setup_method(self, method): + Env.ENVIRON = {} + self.env = Env() + self.env.read_env(Path(__file__, is_file=True)('test_expansion.txt')) + + def test_expansion(self): + assert self.env('HELLO') == 'Hello, world!' + + def test_braces(self): + assert self.env('BRACES') == 'Hello, world!' + + def test_recursion(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('RECURSIVE') + assert str(excinfo.value) == "Environment variable 'RECURSIVE' recursively references itself (eventually)" + + def test_transitive(self): + with pytest.raises(ImproperlyConfigured) as excinfo: + self.env('R4') + assert str(excinfo.value) == "Environment variable 'R4' recursively references itself (eventually)" diff --git a/tests/test_expansion.txt b/tests/test_expansion.txt new file mode 100755 index 00000000..8290e45f --- /dev/null +++ b/tests/test_expansion.txt @@ -0,0 +1,9 @@ +VAR1='Hello' +VAR2='world' +HELLO="$VAR1, $VAR2!" +BRACES="${VAR1}, ${VAR2}!" +RECURSIVE="This variable is $RECURSIVE" +R1="$R2" +R2="$R3" +R3="$R4" +R4="$R1"