Masking configuration values irrelevant to DAG author

airflow/configuration.py

@@ -775,6 +775,13 @@ def _create_future_warning(name: str, section: str, current_value: Any, new_valu
     def _env_var_name(self, section: str, key: str) -> str:
         return f"{ENV_VAR_PREFIX}{section.replace('.', '_').upper()}__{key.upper()}"
 
+    def get_section_and_key_for_env(self, env: str):
+        if env in os.environ:
+            _, section, key = env.split("__", 2)
+            return section, key
+        else:
+            return None, None
+
     def _get_env_var_option(self, section: str, key: str):
         # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
         env_var = self._env_var_name(section, key)

airflow/settings.py

@@ -118,6 +118,27 @@
     "upstream_failed": "orange",
 }
 
+# list of the environment variables that need to be masked from the logs
+# some of these could be sensitive and there's no reason to log those
+ENV_VARIABLES_TO_MASK = [
+    "AIRFLOW__CORE__FERNET_KEY",
+    "AIRFLOW__SMTP__SMTP_PORT",
+    "AIRFLOW__SMTP__SMTP_PASSWORD",
+    "AIRFLOW__SMTP__SMTP_PASSWORD_SECRET",
+    "AIRFLOW__SMTP__SMTP_USER",
+    "AIRFLOW__SMTP__SMTP_SSL",
+    "AIRFLOW__SMTP__SMTP_HOST",
+    "AIRFLOW__WEBSERVER__SECRET_KEY",
+    "AIRFLOW__WEBSERVER__SECRET_KEY_SECRET",
+    "AIRFLOW__SENTRY__SENTRY_DSN_SECRET",
+    "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN_SECRET",
+    "AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS_SECRET",
+    "AIRFLOW__CORE__INTERNAL_API_SECRET_KEY",
+    "AIRFLOW__CORE__INTERNAL_API_SECRET_KEY_SECRET",
+    "AIRFLOW__CORE__DATASET_MANAGER_KWARGS_SECRET",
+    "AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS_SECRET",
+]
+
 
 @functools.lru_cache(maxsize=None)
 def _get_rich_console(file):
@@ -722,6 +743,16 @@ def import_local_settings():
         log.info("Loaded airflow_local_settings from %s .", airflow_local_settings.__file__)
 
 
+def mask_conf_values():
+    from airflow.utils.log.secrets_masker import mask_secret
+
+    for env in ENV_VARIABLES_TO_MASK:
+        section, key = conf.get_section_and_key_for_env(env)
+        if section is None and key is None:
+            continue
+        mask_secret(conf.get(section, key))
+
+
 def initialize():
     """Initialize Airflow with all the settings from this file."""
     configure_vars()
@@ -741,6 +772,9 @@ def initialize():
     configure_orm()
     configure_action_logging()
 
+    # mask the configuration irrelevant to DAG author
+    mask_conf_values()
+
     # Run any custom runtime checks that needs to be executed for providers
     run_providers_custom_runtime_checks()
 

tests/core/test_settings.py

@@ -31,7 +31,12 @@
 from airflow.api_internal.internal_api_call import InternalApiConfig
 from airflow.configuration import conf
 from airflow.exceptions import AirflowClusterPolicyViolation, AirflowConfigException
-from airflow.settings import _ENABLE_AIP_44, TracebackSession, is_usage_data_collection_enabled
+from airflow.settings import (
+    _ENABLE_AIP_44,
+    TracebackSession,
+    is_usage_data_collection_enabled,
+    mask_conf_values,
+)
 from airflow.utils.session import create_session
 from tests_common.test_utils.config import conf_vars
 
@@ -385,3 +390,21 @@ def test_usage_data_collection_disabled(env_var, conf_setting, is_enabled, clear
     else:
         with conf_patch:
             assert is_usage_data_collection_enabled() == is_enabled
+
+
+@patch("airflow.utils.log.secret_masker.mask_secret")
+@patch("airflow.configuration.conf.get")
+@patch("airflow.configuration.get_section_and_key_for_env")
+@patch("airflow.settings.ENV_VARIABLES_TO_MASK", ["AIRFLOW__CORE__FERNET_KEY", "AIRFLOW__NONMASK_ENV"])
+def test_mask_conf_values(self, mock_get_section_and_key, mock_get, mock_mask_secret):
+    mock_get_section_and_key.side_effect = [
+        ("core", "fernet_key"),
+        (None, None),
+    ]
+
+    mock_get.side_effect = ["my_fernet_key"]
+    mask_conf_values()
+
+    mock_mask_secret.assert_called_once_with("my_fernet_key")
+    assert mock_get_section_and_key.call_count == 2
+    mock_get.assert_called_once_with("core", "fernet_key")