From a65216331aca07e918eb38cd6b58001ab99ed601 Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Tue, 3 Dec 2024 14:23:27 +0100 Subject: [PATCH] Improve field value parsing by adding `NoDecode` and `ForceDecode` annotations --- docs/index.md | 82 +++++++++++++++++++++++++++++++++++ pydantic_settings/__init__.py | 4 ++ pydantic_settings/main.py | 2 + pydantic_settings/sources.py | 14 ++++++ tests/test_settings.py | 70 ++++++++++++++++++++++++++++++ 5 files changed, 172 insertions(+) diff --git a/docs/index.md b/docs/index.md index 218da6a..4b0751e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -371,6 +371,88 @@ print(Settings().model_dump()) #> {'numbers': [1, 2, 3]} ``` +### Disabling JSON parsing + +pydatnic-settings by default parses complex types from environment variables as JSON strings. If you want to disable +this behavior for a field and parse the value by your own, you can annotate the field with `NoDecode`: + +```py +import os +from typing import List + +from pydantic import field_validator +from typing_extensions import Annotated + +from pydantic_settings import BaseSettings, NoDecode + + +class Settings(BaseSettings): + numbers: Annotated[List[int], NoDecode] # (1)! + + @field_validator('numbers', mode='before') + @classmethod + def decode_numbers(cls, v: str) -> List[int]: + return [int(x) for x in v.split(',')] + + +os.environ['numbers'] = '1,2,3' +print(Settings().model_dump()) +#> {'numbers': [1, 2, 3]} +``` + +1. The `NoDecode` annotation disables JSON parsing for the `numbers` field. The `decode_numbers` field validator + will be called to parse the value. + +You can also disable JSON parsing for all fields by setting the `enable_decoding` config setting to `False`: + +```py +import os +from typing import List + +from pydantic import field_validator + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(enable_decoding=False) + + numbers: List[int] + + @field_validator('numbers', mode='before') + @classmethod + def decode_numbers(cls, v: str) -> List[int]: + return [int(x) for x in v.split(',')] + + +os.environ['numbers'] = '1,2,3' +print(Settings().model_dump()) +#> {'numbers': [1, 2, 3]} +``` + +You can force JSON parsing for a field by annotating it with `ForceDecode`. This will bypass +the the `enable_decoding` config setting: + +```py +import os +from typing import List + +from typing_extensions import Annotated + +from pydantic_settings import BaseSettings, ForceDecode, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(enable_decoding=False) + + numbers: Annotated[List[int], ForceDecode] + + +os.environ['numbers'] = '["1","2","3"]' +print(Settings().model_dump()) +#> {'numbers': [1, 2, 3]} +``` + ## Nested model default partial updates By default, Pydantic settings does not allow partial updates to nested model default objects. This behavior can be diff --git a/pydantic_settings/__init__.py b/pydantic_settings/__init__.py index 5b3aa9f..0a02868 100644 --- a/pydantic_settings/__init__.py +++ b/pydantic_settings/__init__.py @@ -11,8 +11,10 @@ CliSuppress, DotEnvSettingsSource, EnvSettingsSource, + ForceDecode, InitSettingsSource, JsonConfigSettingsSource, + NoDecode, PydanticBaseSettingsSource, PyprojectTomlConfigSettingsSource, SecretsSettingsSource, @@ -38,6 +40,8 @@ 'CliMutuallyExclusiveGroup', 'InitSettingsSource', 'JsonConfigSettingsSource', + 'NoDecode', + 'ForceDecode', 'PyprojectTomlConfigSettingsSource', 'PydanticBaseSettingsSource', 'SecretsSettingsSource', diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 4903ffc..c695f28 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -78,6 +78,7 @@ class SettingsConfigDict(ConfigDict, total=False): """ toml_file: PathType | None + enable_decoding: bool # Extend `config_keys` by pydantic settings config keys to @@ -425,6 +426,7 @@ def _settings_build_values( toml_file=None, secrets_dir=None, protected_namespaces=('model_validate', 'model_dump', 'settings_customise_sources'), + enable_decoding=True, ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 9fe3f03..937fd38 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -118,6 +118,14 @@ def import_azure_key_vault() -> None: ENV_FILE_SENTINEL: DotenvType = Path('') +class NoDecode: + pass + + +class ForceDecode: + pass + + class SettingsError(ValueError): pass @@ -312,6 +320,12 @@ def decode_complex_value(self, field_name: str, field: FieldInfo, value: Any) -> Returns: The decoded value for further preparation """ + if field and ( + NoDecode in field.metadata + or (self.config.get('enable_decoding') is False and ForceDecode not in field.metadata) + ): + return value + return json.loads(value) @abstractmethod diff --git a/tests/test_settings.py b/tests/test_settings.py index 143b285..2a6578b 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,4 +1,5 @@ import dataclasses +import json import os import pathlib import sys @@ -26,6 +27,7 @@ SecretStr, Tag, ValidationError, + field_validator, ) from pydantic import ( dataclasses as pydantic_dataclasses, @@ -37,7 +39,9 @@ BaseSettings, DotEnvSettingsSource, EnvSettingsSource, + ForceDecode, InitSettingsSource, + NoDecode, PydanticBaseSettingsSource, SecretsSettingsSource, SettingsConfigDict, @@ -2873,3 +2877,69 @@ class Settings(BaseSettings): s = Settings() assert s.foo.get_secret_value() == 123 assert s.bar.get_secret_value() == PostgresDsn('postgres://user:password@localhost/dbname') + + +def test_field_annotated_no_decode(env): + class Settings(BaseSettings): + a: List[str] # this field will be decoded because of default `enable_decoding=True` + b: Annotated[List[str], NoDecode] + + # decode the value here. the field value won't be decoded because of NoDecode + @field_validator('b', mode='before') + @classmethod + def decode_b(cls, v: str) -> List[str]: + return json.loads(v) + + env.set('a', '["one", "two"]') + env.set('b', '["1", "2"]') + + s = Settings() + assert s.model_dump() == {'a': ['one', 'two'], 'b': ['1', '2']} + + +def test_field_annotated_no_decode_and_disable_decoding(env): + class Settings(BaseSettings): + model_config = SettingsConfigDict(enable_decoding=False) + + a: Annotated[List[str], NoDecode] + + # decode the value here. the field value won't be decoded because of NoDecode + @field_validator('a', mode='before') + @classmethod + def decode_b(cls, v: str) -> List[str]: + return json.loads(v) + + env.set('a', '["one", "two"]') + + s = Settings() + assert s.model_dump() == {'a': ['one', 'two']} + + +def test_field_annotated_disable_decoding(env): + class Settings(BaseSettings): + model_config = SettingsConfigDict(enable_decoding=False) + + a: List[str] + + # decode the value here. the field value won't be decoded because of `enable_decoding=False` + @field_validator('a', mode='before') + @classmethod + def decode_b(cls, v: str) -> List[str]: + return json.loads(v) + + env.set('a', '["one", "two"]') + + s = Settings() + assert s.model_dump() == {'a': ['one', 'two']} + + +def test_field_annotated_force_decode_disable_decoding(env): + class Settings(BaseSettings): + model_config = SettingsConfigDict(enable_decoding=False) + + a: Annotated[List[str], ForceDecode] + + env.set('a', '["one", "two"]') + + s = Settings() + assert s.model_dump() == {'a': ['one', 'two']}