added potentially controversial support for using real keys in tests

plugins/validation_tests/test_object_creation.py

@@ -20,7 +20,8 @@
 
 # Ensure all the plugins are available during testing.
 load_plugins()
-_FAKE_SECRETS = fake_all_secrets()
+# Some tests need to download a file from modellab, which requires a real auth token
+_FAKE_SECRETS = fake_all_secrets(use_real_secrets_for=("modellab_files",))
 
 
 @pytest.mark.parametrize("test_name", [key for key, _ in TESTS.items()])

src/modelgauge/secret_values.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+from typing import Generic, List, Mapping, Optional, Sequence, Type, TypeVar
+
 from modelgauge.general import get_concrete_subclasses
 from pydantic import BaseModel
-from typing import Generic, List, Mapping, Optional, Sequence, Type, TypeVar
 
 
 class SecretDescription(BaseModel):

tests/modelgauge_tests/fake_secrets.py

@@ -1,23 +1,27 @@
-from modelgauge.secret_values import (
-    RawSecrets,
-    RequiredSecret,
-    SecretDescription,
-    get_all_secrets,
-)
 from typing import Dict
 
+from modelgauge.config import load_secrets_from_config
+
+from modelgauge.secret_values import get_all_secrets, RawSecrets, RequiredSecret, SecretDescription
+
 
 class FakeRequiredSecret(RequiredSecret):
     @classmethod
     def description(cls) -> SecretDescription:
         return SecretDescription(scope="some-scope", key="some-key", instructions="some-instructions")
 
 
-def fake_all_secrets(value="some-value") -> RawSecrets:
+def fake_all_secrets(value="some-value", use_real_secrets_for: list[str] | None = None) -> RawSecrets:
     secrets = get_all_secrets()
     raw_secrets: Dict[str, Dict[str, str]] = {}
+    real_secrets = load_secrets_from_config()
+
     for secret in secrets:
         if secret.scope not in raw_secrets:
             raw_secrets[secret.scope] = {}
-        raw_secrets[secret.scope][secret.key] = value
+        if use_real_secrets_for and secret.scope in use_real_secrets_for:
+            raw_secrets[secret.scope][secret.key] = real_secrets[secret.scope][secret.key]
+        else:
+            raw_secrets[secret.scope][secret.key] = value
+
     return raw_secrets