From 64379cf24902beb50e068e9ded2fc0a30e214860 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Thu, 26 Oct 2023 17:45:50 +0100 Subject: [PATCH 01/29] changes to core --- .../core/openbb_core/app/model/credentials.py | 81 +++++++++++++++---- .../core/openbb_core/app/model/obbject.py | 15 +++- .../core/openbb_core/app/static/container.py | 1 + .../core/openbb_core/app/static/decorators.py | 74 ++++++++++++++++- 4 files changed, 153 insertions(+), 18 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index d8d0132fa394..0650cec06184 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -1,6 +1,12 @@ -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple -from pydantic import ConfigDict, SecretStr, create_model, field_serializer +from pydantic import ( + ConfigDict, + SecretStr, + create_model, + model_serializer, +) +from pydantic.fields import FieldInfo from openbb_core.app.provider_interface import ProviderInterface @@ -32,9 +38,63 @@ def format_map( class Credentials(_Credentials): """Credentials model used to store provider credentials""" - @field_serializer(*provider_credentials, when_used="json-unless-none") - def _dump_secret(self, v): - return v.get_secret_value() + def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + """Override model_dump to include new fields added with `_add_fields`""" + instance_fields = super().model_dump(*args, **kwargs) + class_fields = self.model_fields + for f_name, f_info in class_fields.items(): + instance_fields.setdefault(f_name, f_info.default) + + return instance_fields + + @model_serializer(when_used="json-unless-none") + def _serialize(self) -> Dict[str, Any]: + return { + k: v.get_secret_value() if isinstance(v, SecretStr) else v + for k, v in self.model_dump().items() + } + + def __repr__(self) -> str: + return ( + self.__class__.__name__ + + "\n\n" + + "\n".join([f"{k}: {v}" for k, v in self.model_dump().items()]) + ) + + @classmethod + def _add_fields(cls, **field_definitions: Any) -> None: + """Add new fields to the Credentials model""" + new_fields: Dict[str, FieldInfo] = {} + new_annotations: Dict[str, Optional[type]] = {} + + for f_name, f_def in field_definitions.items(): + if isinstance(f_def, tuple): + try: + f_annotation, f_value = f_def + except ValueError as e: + raise Exception( + "field definitions should either be a tuple of (, ) or just a " + "default value, unfortunately this means tuples as " + "default values are not allowed" + ) from e + else: + f_annotation, f_value = None, f_def + + if f_annotation: + new_annotations[f_name] = f_annotation + + new_fields[f_name] = FieldInfo.from_annotated_attribute( + annotation=f_annotation, default=f_value + ) + + if hasattr(cls, f_name): + raise ValueError(f"Attribute '{f_name}' is already defined.") + setattr(cls, f_name, None) + + cls.model_fields.update(new_fields) + cls.__annotations__.update(new_annotations) + cls.model_rebuild(force=True) + def show(self): """Unmask credentials and print them""" @@ -43,14 +103,3 @@ def show(self): + "\n\n" + "\n".join([f"{k}: {v}" for k, v in self.model_dump(mode="json").items()]) ) - - -def __repr__(self: Credentials) -> str: - return ( - self.__class__.__name__ - + "\n\n" - + "\n".join([f"{k}: {v}" for k, v in self.model_dump().items()]) - ) - - -Credentials.__repr__ = __repr__ diff --git a/openbb_platform/platform/core/openbb_core/app/model/obbject.py b/openbb_platform/platform/core/openbb_core/app/model/obbject.py index 1296d4faca11..6ee58974f05a 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/obbject.py +++ b/openbb_platform/platform/core/openbb_core/app/model/obbject.py @@ -1,6 +1,17 @@ """The OBBject.""" from re import sub -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Literal, Optional, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Dict, + Generic, + List, + Literal, + Optional, + Set, + TypeVar, +) import pandas as pd from numpy import ndarray @@ -47,6 +58,8 @@ class OBBject(Tagged, Generic[T]): default_factory=dict, description="Extra info.", ) + _credentials: BaseModel + _accessors: ClassVar[Set[str]] = set() def __repr__(self) -> str: """Human readable representation of the object.""" diff --git a/openbb_platform/platform/core/openbb_core/app/static/container.py b/openbb_platform/platform/core/openbb_core/app/static/container.py index 6f5c43205643..c69bbba13054 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/container.py +++ b/openbb_platform/platform/core/openbb_core/app/static/container.py @@ -12,6 +12,7 @@ class Container: def __init__(self, command_runner: CommandRunner) -> None: self._command_runner = command_runner + OBBject._credentials = command_runner.user_settings.credentials def _run(self, *args, **kwargs) -> Union[OBBject, pd.DataFrame, dict]: """Run a command in the container.""" diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index 8d3798647241..f964b78714fb 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -1,9 +1,13 @@ +import warnings from functools import wraps -from typing import Any, Callable, Optional, TypeVar, overload +from typing import Any, Callable, List, Optional, TypeVar, overload from pydantic.validate_call import validate_call from typing_extensions import ParamSpec +from openbb_core.app.model.credentials import Credentials, format_map +from openbb_core.app.model.obbject import OBBject + P = ParamSpec("P") R = TypeVar("R") @@ -34,3 +38,71 @@ def wrapper(*f_args, **f_kwargs): return wrapper return decorated if func is None else decorated(func) + + +class CachedAccessor: + """CachedAccessor""" + + def __init__(self, name: str, accessor) -> None: + self._name = name + self._accessor = accessor + + def __get__(self, obj, cls): + if obj is None: + return self._accessor + accessor_obj = self._accessor(obj) + object.__setattr__(obj, self._name, accessor_obj) + return accessor_obj + + +def _register_accessor(name, cls) -> Callable: + """Register a custom accessor""" + + def decorator(accessor): + if hasattr(cls, name): + warnings.warn( + f"registration of accessor '{repr(accessor)}' under name " + f"'{repr(name)}' for type '{repr(cls)}' is overriding a preexisting " + f"attribute with the same name.", + UserWarning, + ) + setattr(cls, name, CachedAccessor(name, accessor)) + # pylint: disable=protected-access + cls._accessors.add(name) + return accessor + + return decorator + + +def extend_obbject(name: str, required_credentials: List[str]) -> Callable: + """Extend an OBBject, inspired by pandas. + + Parameters + ---------- + name : str + Name of the extension. + + required_credentials : List[str] + List of required credentials. + + Returns + ------- + Callable + Decorator for the extension. + + Example + ------- + @extend_obbject(name="useless", required_credentials=["api_key"]) + class Useless: + def __init__(self, obbject): + self._obbject = obbject + + def hello(self) -> str: + cred = self._obbject._credentials.model_dump(mode="json")["useless_api_key"] + return f"Hi, I'm {self.__class__.__name__}, this is my credential: {cred}!" + + """ + formatted_creds = [f"{name}_{c}" for c in required_credentials] + # pylint: disable=protected-access + Credentials._add_fields(**format_map(formatted_creds)) + return _register_accessor(name, OBBject) From df2cd4c2879b0e6ac8492be421eb7e59ff0f5afe Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Thu, 26 Oct 2023 18:03:39 +0100 Subject: [PATCH 02/29] bug? --- .../platform/core/openbb_core/app/model/credentials.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index 0650cec06184..19bbc08b3457 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -82,10 +82,9 @@ def _add_fields(cls, **field_definitions: Any) -> None: if f_annotation: new_annotations[f_name] = f_annotation - - new_fields[f_name] = FieldInfo.from_annotated_attribute( - annotation=f_annotation, default=f_value - ) + new_fields[f_name] = FieldInfo.from_annotated_attribute( + annotation=f_annotation, default=f_value + ) if hasattr(cls, f_name): raise ValueError(f"Attribute '{f_name}' is already defined.") @@ -95,7 +94,6 @@ def _add_fields(cls, **field_definitions: Any) -> None: cls.__annotations__.update(new_annotations) cls.model_rebuild(force=True) - def show(self): """Unmask credentials and print them""" print( # noqa: T201 From bb4a7d41c3c053109d024ba709fd36458a59ffe5 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Thu, 26 Oct 2023 18:05:54 +0100 Subject: [PATCH 03/29] docstring --- .../platform/core/openbb_core/app/static/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index f964b78714fb..247c8215bf56 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -80,7 +80,7 @@ def extend_obbject(name: str, required_credentials: List[str]) -> Callable: Parameters ---------- name : str - Name of the extension. + Name of the accessor. required_credentials : List[str] List of required credentials. From eb2b47909dac91dc1cdc2d9d9de0e9a2fa8fa4d8 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Thu, 26 Oct 2023 18:11:54 +0100 Subject: [PATCH 04/29] doc --- .../core/openbb_core/app/static/decorators.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index 247c8215bf56..2cbd6f425738 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -92,15 +92,18 @@ def extend_obbject(name: str, required_credentials: List[str]) -> Callable: Example ------- - @extend_obbject(name="useless", required_credentials=["api_key"]) - class Useless: - def __init__(self, obbject): - self._obbject = obbject - - def hello(self) -> str: - cred = self._obbject._credentials.model_dump(mode="json")["useless_api_key"] + @extend_obbject(name="useless", required_credentials=["api_key"] + class Useless + def __init__(self, obbject) + self._obbject = obbj + def hello(self) -> str + cred = self._obbject._credentials.model_dump(mode="json")["useless_api_key" return f"Hi, I'm {self.__class__.__name__}, this is my credential: {cred}!" + >>> from openbb import obb + >>> obbject = obb.stocks.load("AAPL") + >>> obbject.useless.hello() + Hi, I'm Useless, this is my credential: None! """ formatted_creds = [f"{name}_{c}" for c in required_credentials] # pylint: disable=protected-access From f2d5b7320d236a5c77772de1fd9050f22ebb7c87 Mon Sep 17 00:00:00 2001 From: montezdesousa <79287829+montezdesousa@users.noreply.github.com> Date: Thu, 26 Oct 2023 18:45:37 +0100 Subject: [PATCH 05/29] Update credentials.py --- .../platform/core/openbb_core/app/model/credentials.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index 19bbc08b3457..c632caa6e81a 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -82,9 +82,9 @@ def _add_fields(cls, **field_definitions: Any) -> None: if f_annotation: new_annotations[f_name] = f_annotation - new_fields[f_name] = FieldInfo.from_annotated_attribute( - annotation=f_annotation, default=f_value - ) + new_fields[f_name] = FieldInfo.from_annotated_attribute( + annotation=f_annotation, default=f_value + ) if hasattr(cls, f_name): raise ValueError(f"Attribute '{f_name}' is already defined.") From f6bd6cb985561e1edef4c2acf8460f063949d762 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 13:03:21 +0100 Subject: [PATCH 06/29] fix model_dump --- .../core/openbb_core/app/model/credentials.py | 34 +++++++++---------- .../openbb_core/app/model/user_settings.py | 4 ++- .../openbb_provider/query_executor.py | 8 ++--- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index c632caa6e81a..dfd7c57cc862 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -38,24 +38,32 @@ def format_map( class Credentials(_Credentials): """Credentials model used to store provider credentials""" - def model_dump(self, *args, **kwargs) -> Dict[str, Any]: - """Override model_dump to include new fields added with `_add_fields`""" - instance_fields = super().model_dump(*args, **kwargs) + @model_serializer(when_used="always") + def _serialize(self) -> Dict[str, Any]: + """Serialize credentials to a dict""" + # We override the default serializer to include new_fields added after init + # This will be called everytime model_dump() is called + instance_fields = vars(super()) class_fields = self.model_fields for f_name, f_info in class_fields.items(): instance_fields.setdefault(f_name, f_info.default) - - return instance_fields - - @model_serializer(when_used="json-unless-none") - def _serialize(self) -> Dict[str, Any]: return { k: v.get_secret_value() if isinstance(v, SecretStr) else v - for k, v in self.model_dump().items() + for k, v in instance_fields.items() } def __repr__(self) -> str: + # We use the __dict__ because model_dump() will use the serializer + # and unmask the credentials return ( + self.__class__.__name__ + + "\n\n" + + "\n".join([f"{k}: {v}" for k, v in self.__dict__.items()]) + ) + + def show(self): + """Unmask credentials and print them""" + print( # noqa: T201 self.__class__.__name__ + "\n\n" + "\n".join([f"{k}: {v}" for k, v in self.model_dump().items()]) @@ -93,11 +101,3 @@ def _add_fields(cls, **field_definitions: Any) -> None: cls.model_fields.update(new_fields) cls.__annotations__.update(new_annotations) cls.model_rebuild(force=True) - - def show(self): - """Unmask credentials and print them""" - print( # noqa: T201 - self.__class__.__name__ - + "\n\n" - + "\n".join([f"{k}: {v}" for k, v in self.model_dump(mode="json").items()]) - ) diff --git a/openbb_platform/platform/core/openbb_core/app/model/user_settings.py b/openbb_platform/platform/core/openbb_core/app/model/user_settings.py index 7ba9fe68706d..11795515bb2d 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/user_settings.py +++ b/openbb_platform/platform/core/openbb_core/app/model/user_settings.py @@ -14,6 +14,8 @@ class UserSettings(Tagged): defaults: Defaults = Field(default_factory=Defaults) def __repr__(self) -> str: + # We use the __dict__ because Credentials.model_dump() will use the serializer + # and unmask the credentials return f"{self.__class__.__name__}\n\n" + "\n".join( - f"{k}: {v}" for k, v in self.model_dump().items() + f"{k}: {v}" for k, v in self.__dict__.items() ) diff --git a/openbb_platform/platform/provider/openbb_provider/query_executor.py b/openbb_platform/platform/provider/openbb_provider/query_executor.py index 9990a351338e..d13dea0b5081 100644 --- a/openbb_platform/platform/provider/openbb_provider/query_executor.py +++ b/openbb_platform/platform/provider/openbb_provider/query_executor.py @@ -1,8 +1,6 @@ """Query executor module.""" from typing import Any, Dict, Optional, Type -from pydantic import SecretStr - from openbb_provider.abstract.fetcher import Fetcher from openbb_provider.abstract.provider import Provider from openbb_provider.registry import Registry, RegistryLoader @@ -36,7 +34,7 @@ def get_fetcher(self, provider: Provider, model_name: str) -> Type[Fetcher]: @staticmethod def filter_credentials( - provider: Provider, credentials: Optional[Dict[str, SecretStr]] + provider: Provider, credentials: Optional[Dict[str, str]] ) -> Dict[str, str]: """Filter credentials and check if they match provider requirements.""" if provider.required_credentials is not None: @@ -48,7 +46,7 @@ def filter_credentials( credential_value = credentials.get(c) if c not in credentials or credential_value is None: raise ProviderError(f"Missing credential '{c}'.") - filtered_credentials[c] = credential_value.get_secret_value() + filtered_credentials[c] = credential_value return filtered_credentials @@ -57,7 +55,7 @@ def execute( provider_name: str, model_name: str, params: Dict[str, Any], - credentials: Optional[Dict[str, SecretStr]] = None, + credentials: Optional[Dict[str, str]] = None, **kwargs: Any, ) -> Any: """Execute query. From 862ae606b57b961c90044b53c8aba95a41176204 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 23:08:05 +0100 Subject: [PATCH 07/29] create extensions --- .../core/openbb_core/app/model/credentials.py | 124 ++++++++---------- .../core/openbb_core/app/model/extension.py | 27 ++++ .../core/openbb_core/app/model/obbject.py | 4 +- .../core/openbb_core/app/static/decorators.py | 67 +++++----- 4 files changed, 118 insertions(+), 104 deletions(-) create mode 100644 openbb_platform/platform/core/openbb_core/app/model/extension.py diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index dfd7c57cc862..f579c633fa37 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -1,60 +1,83 @@ -from typing import Any, Dict, List, Optional, Tuple +import traceback +from typing import Any, Dict, Optional, Set, Tuple +from importlib_metadata import entry_points from pydantic import ( + BaseModel, ConfigDict, SecretStr, create_model, model_serializer, ) -from pydantic.fields import FieldInfo +from openbb_core.app.model.extension import Extension from openbb_core.app.provider_interface import ProviderInterface -# Here we create the BaseModel from the provider required credentials. -# This means that if a new provider extension is installed, the required -# credentials will be automatically added to the Credentials model. - - -def format_map( - required_credentials: List[str], -) -> Dict[str, Tuple[object, None]]: - """Format credentials map to be used in the Credentials model""" - formatted: Dict[str, Tuple[object, None]] = {} - for c in required_credentials: - formatted[c] = (Optional[SecretStr], None) - - return formatted +class LoadingError(Exception): + """Error loading extension.""" + + +class CredentialsLoader: + """Here we create the Credentials model from the provider required credentials""" + + credentials: Set[str] = set() + + @staticmethod + def prepare( + required_credentials: Set[str], + ) -> Dict[str, Tuple[object, None]]: + """Prepare credentials map to be used in the Credentials model""" + formatted: Dict[str, Tuple[object, None]] = {} + for c in required_credentials: + formatted[c] = (Optional[SecretStr], None) + + return formatted + + def from_providers(self) -> None: + """Load credentials from providers""" + for c in ProviderInterface().required_credentials: + self.credentials.add(c) + + def from_extensions(self) -> None: + """Load credentials from extensions""" + for entry_point in sorted(entry_points(group="openbb_obbject_extension")): + try: + entry = entry_point.load() + if isinstance(entry, Extension): + for c in entry.required_credentials: + self.credentials.add(c) + except Exception as e: + traceback.print_exception(type(e), e, e.__traceback__) + raise LoadingError(f"Invalid extension '{entry_point.name}'") from e + + def load(self) -> BaseModel: + """Load credentials from providers""" + self.from_providers() + self.from_extensions() + return create_model( # type: ignore + "Credentials", + __config__=ConfigDict(validate_assignment=True), + **self.prepare(self.credentials), + ) -provider_credentials = ProviderInterface().required_credentials -_Credentials = create_model( # type: ignore - "Credentials", - __config__=ConfigDict(validate_assignment=True), - **format_map(provider_credentials), -) +_Credentials = CredentialsLoader().load() -class Credentials(_Credentials): +class Credentials(_Credentials): # type: ignore """Credentials model used to store provider credentials""" - @model_serializer(when_used="always") + @model_serializer(when_used="json-unless-none") def _serialize(self) -> Dict[str, Any]: """Serialize credentials to a dict""" - # We override the default serializer to include new_fields added after init - # This will be called everytime model_dump() is called - instance_fields = vars(super()) - class_fields = self.model_fields - for f_name, f_info in class_fields.items(): - instance_fields.setdefault(f_name, f_info.default) return { k: v.get_secret_value() if isinstance(v, SecretStr) else v - for k, v in instance_fields.items() + for k, v in self.__dict__.items() } def __repr__(self) -> str: - # We use the __dict__ because model_dump() will use the serializer - # and unmask the credentials + """String representation of the credentials""" return ( self.__class__.__name__ + "\n\n" @@ -66,38 +89,5 @@ def show(self): print( # noqa: T201 self.__class__.__name__ + "\n\n" - + "\n".join([f"{k}: {v}" for k, v in self.model_dump().items()]) + + "\n".join([f"{k}: {v}" for k, v in self.model_dump(mode="json").items()]) ) - - @classmethod - def _add_fields(cls, **field_definitions: Any) -> None: - """Add new fields to the Credentials model""" - new_fields: Dict[str, FieldInfo] = {} - new_annotations: Dict[str, Optional[type]] = {} - - for f_name, f_def in field_definitions.items(): - if isinstance(f_def, tuple): - try: - f_annotation, f_value = f_def - except ValueError as e: - raise Exception( - "field definitions should either be a tuple of (, ) or just a " - "default value, unfortunately this means tuples as " - "default values are not allowed" - ) from e - else: - f_annotation, f_value = None, f_def - - if f_annotation: - new_annotations[f_name] = f_annotation - new_fields[f_name] = FieldInfo.from_annotated_attribute( - annotation=f_annotation, default=f_value - ) - - if hasattr(cls, f_name): - raise ValueError(f"Attribute '{f_name}' is already defined.") - setattr(cls, f_name, None) - - cls.model_fields.update(new_fields) - cls.__annotations__.update(new_annotations) - cls.model_rebuild(force=True) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py new file mode 100644 index 000000000000..f256c4f9803d --- /dev/null +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -0,0 +1,27 @@ +from typing import List, Optional + + +class Extension: + """Serves as extension entry point and must be created by each provider.""" + + def __init__( + self, + name: str, + required_credentials: Optional[List[str]] = None, + ) -> None: + """Initialize the extension. + + Parameters + ---------- + name : str + Name of the provider. + required_credentials : Optional[List[str]], optional + List of required credentials, by default None + """ + self.name = name + if required_credentials is None: + self.required_credentials: List = [] + else: + self.required_credentials = [] + for rq in required_credentials: + self.required_credentials.append(f"{self.name.lower()}_{rq}") diff --git a/openbb_platform/platform/core/openbb_core/app/model/obbject.py b/openbb_platform/platform/core/openbb_core/app/model/obbject.py index 6ee58974f05a..2917b6fb53a6 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/obbject.py +++ b/openbb_platform/platform/core/openbb_core/app/model/obbject.py @@ -17,7 +17,6 @@ from numpy import ndarray from pydantic import BaseModel, Field -from openbb_core.app.charting_service import ChartingService from openbb_core.app.model.abstract.error import OpenBBError from openbb_core.app.model.abstract.tagged import Tagged from openbb_core.app.model.abstract.warning import Warning_ @@ -249,6 +248,9 @@ def to_chart(self, **kwargs): chart.fig The chart figure. """ + # pylint: disable=import-outside-toplevel + # Avoids circular import + from openbb_core.app.charting_service import ChartingService cs = ChartingService() kwargs["data"] = self.to_dataframe() diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index 2cbd6f425738..d99dbde49451 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -1,13 +1,10 @@ import warnings from functools import wraps -from typing import Any, Callable, List, Optional, TypeVar, overload +from typing import Any, Callable, Optional, TypeVar, overload from pydantic.validate_call import validate_call from typing_extensions import ParamSpec -from openbb_core.app.model.credentials import Credentials, format_map -from openbb_core.app.model.obbject import OBBject - P = ParamSpec("P") R = TypeVar("R") @@ -55,7 +52,7 @@ def __get__(self, obj, cls): return accessor_obj -def _register_accessor(name, cls) -> Callable: +def register_accessor(name, cls) -> Callable: """Register a custom accessor""" def decorator(accessor): @@ -74,38 +71,36 @@ def decorator(accessor): return decorator -def extend_obbject(name: str, required_credentials: List[str]) -> Callable: +def extend_obbject(name: str) -> Callable: """Extend an OBBject, inspired by pandas. - Parameters - ---------- - name : str - Name of the accessor. - - required_credentials : List[str] - List of required credentials. - - Returns - ------- - Callable - Decorator for the extension. - - Example - ------- - @extend_obbject(name="useless", required_credentials=["api_key"] - class Useless - def __init__(self, obbject) - self._obbject = obbj - def hello(self) -> str - cred = self._obbject._credentials.model_dump(mode="json")["useless_api_key" - return f"Hi, I'm {self.__class__.__name__}, this is my credential: {cred}!" - + Set the following as entry_point in your extension .toml file: + [tool.poetry.plugins."openbb_obbject_extension"] + useless = "openbb_useless:entry_point" + + Extension code: + ```python + from openbb_core.app.model.extension import Extension + from openbb_core.app.static.decorators import extend_obbject + + entry_point = Extension(name="example", required_credentials=["api_key"]) + @extend_obbject(name="example") + class Example: + def __init__(self, obbject): + self._obbject = obbject + def hello(self): + api_key = self._obbject._credentials.example_api_key + print(f"Hello, this is my credential: {api_key}!") + ``` + + Usage: >>> from openbb import obb - >>> obbject = obb.stocks.load("AAPL") - >>> obbject.useless.hello() - Hi, I'm Useless, this is my credential: None! + >>> obbject = obb.stock.load("AAPL") + >>> obbject.example.hello() + Hello, this is my credential: None! """ - formatted_creds = [f"{name}_{c}" for c in required_credentials] - # pylint: disable=protected-access - Credentials._add_fields(**format_map(formatted_creds)) - return _register_accessor(name, OBBject) + # pylint: disable=import-outside-toplevel + # Avoid circular imports + from openbb_core.app.model.obbject import OBBject + + return register_accessor(name, OBBject) From b059cb34de2b8f0bfd65fabbfed9f4807d8a2432 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 23:12:21 +0100 Subject: [PATCH 08/29] fix docstring --- .../platform/core/openbb_core/app/static/decorators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index d99dbde49451..cbf4b9223076 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -74,9 +74,9 @@ def decorator(accessor): def extend_obbject(name: str) -> Callable: """Extend an OBBject, inspired by pandas. - Set the following as entry_point in your extension .toml file: + Set the following as entry_point in your extension .toml file and install it: [tool.poetry.plugins."openbb_obbject_extension"] - useless = "openbb_useless:entry_point" + example = "openbb_example:entry_point" Extension code: ```python From 31710da93c5b2a680155c3ab40aa64efd6284d7e Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 23:12:48 +0100 Subject: [PATCH 09/29] doc --- .../platform/core/openbb_core/app/static/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index cbf4b9223076..e5b5f57a6b60 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -88,6 +88,7 @@ def extend_obbject(name: str) -> Callable: class Example: def __init__(self, obbject): self._obbject = obbject + def hello(self): api_key = self._obbject._credentials.example_api_key print(f"Hello, this is my credential: {api_key}!") From 81c9c478d5c5e6569b47029b925e42055da1fa31 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 23:15:30 +0100 Subject: [PATCH 10/29] revert change query_exc --- .../platform/provider/openbb_provider/query_executor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openbb_platform/platform/provider/openbb_provider/query_executor.py b/openbb_platform/platform/provider/openbb_provider/query_executor.py index d13dea0b5081..9990a351338e 100644 --- a/openbb_platform/platform/provider/openbb_provider/query_executor.py +++ b/openbb_platform/platform/provider/openbb_provider/query_executor.py @@ -1,6 +1,8 @@ """Query executor module.""" from typing import Any, Dict, Optional, Type +from pydantic import SecretStr + from openbb_provider.abstract.fetcher import Fetcher from openbb_provider.abstract.provider import Provider from openbb_provider.registry import Registry, RegistryLoader @@ -34,7 +36,7 @@ def get_fetcher(self, provider: Provider, model_name: str) -> Type[Fetcher]: @staticmethod def filter_credentials( - provider: Provider, credentials: Optional[Dict[str, str]] + provider: Provider, credentials: Optional[Dict[str, SecretStr]] ) -> Dict[str, str]: """Filter credentials and check if they match provider requirements.""" if provider.required_credentials is not None: @@ -46,7 +48,7 @@ def filter_credentials( credential_value = credentials.get(c) if c not in credentials or credential_value is None: raise ProviderError(f"Missing credential '{c}'.") - filtered_credentials[c] = credential_value + filtered_credentials[c] = credential_value.get_secret_value() return filtered_credentials @@ -55,7 +57,7 @@ def execute( provider_name: str, model_name: str, params: Dict[str, Any], - credentials: Optional[Dict[str, str]] = None, + credentials: Optional[Dict[str, SecretStr]] = None, **kwargs: Any, ) -> Any: """Execute query. From 5599d5379704a99bf7a0d96330658181921a4ea4 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Fri, 27 Oct 2023 23:18:14 +0100 Subject: [PATCH 11/29] doc --- .../platform/core/openbb_core/app/model/credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index f579c633fa37..1d6d0964c664 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -19,7 +19,7 @@ class LoadingError(Exception): class CredentialsLoader: - """Here we create the Credentials model from the provider required credentials""" + """Here we create the Credentials model""" credentials: Set[str] = set() From 14aacbc800a98d320479d4648144311a57184a5a Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 10:29:48 +0000 Subject: [PATCH 12/29] fix container test --- .../platform/core/tests/app/static/test_container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/tests/app/static/test_container.py b/openbb_platform/platform/core/tests/app/static/test_container.py index dc1e01b28010..dd2d7df0eff5 100644 --- a/openbb_platform/platform/core/tests/app/static/test_container.py +++ b/openbb_platform/platform/core/tests/app/static/test_container.py @@ -1,9 +1,10 @@ """Test the container.py file.""" +from openbb_core.app.command_runner import CommandRunner from openbb_core.app.static.container import Container def test_container_init(): """Test container init.""" - container = Container(None) + container = Container(CommandRunner()) assert container From b27e1710f13284e90b3715518a0ecaf2440ec261 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 10:34:10 +0000 Subject: [PATCH 13/29] redirect obbject test patch --- openbb_platform/platform/core/tests/app/model/test_obbject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/tests/app/model/test_obbject.py b/openbb_platform/platform/core/tests/app/model/test_obbject.py index 0f8868226322..37148b3c0af1 100644 --- a/openbb_platform/platform/core/tests/app/model/test_obbject.py +++ b/openbb_platform/platform/core/tests/app/model/test_obbject.py @@ -284,7 +284,7 @@ def test_to_dict(results, expected_dict): @patch("openbb_core.app.model.obbject.OBBject.to_dataframe") -@patch("openbb_core.app.model.obbject.ChartingService") +@patch("openbb_core.app.charting_service.ChartingService") def test_to_chart_with_new_chart( mock_charting_service, mock_to_dataframe, From 4ef513c5b4f29ef4fc51cbf21e07372fec4ac62e Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 10:36:16 +0000 Subject: [PATCH 14/29] doc --- .../platform/core/openbb_core/app/model/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index f256c4f9803d..077d10b0858f 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -2,7 +2,7 @@ class Extension: - """Serves as extension entry point and must be created by each provider.""" + """Serves as extension entry point and must be created by each extension package.""" def __init__( self, From 7d78cbdc48237197b2959052c7416416de0b5e18 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 10:43:33 +0000 Subject: [PATCH 15/29] rename method --- .../core/openbb_core/app/model/credentials.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index 1d6d0964c664..ef03f69ea63a 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -34,13 +34,8 @@ def prepare( return formatted - def from_providers(self) -> None: - """Load credentials from providers""" - for c in ProviderInterface().required_credentials: - self.credentials.add(c) - - def from_extensions(self) -> None: - """Load credentials from extensions""" + def from_obbject(self) -> None: + """Load credentials from OBBject extensions""" for entry_point in sorted(entry_points(group="openbb_obbject_extension")): try: entry = entry_point.load() @@ -51,10 +46,20 @@ def from_extensions(self) -> None: traceback.print_exception(type(e), e, e.__traceback__) raise LoadingError(f"Invalid extension '{entry_point.name}'") from e + def from_providers(self) -> None: + """Load credentials from providers""" + for c in ProviderInterface().required_credentials: + self.credentials.add(c) + + def from_routers(self) -> None: + """Load credentials from routers""" + # Placeholder for future use if needed + def load(self) -> BaseModel: """Load credentials from providers""" + self.from_obbject() self.from_providers() - self.from_extensions() + self.from_routers() return create_model( # type: ignore "Credentials", __config__=ConfigDict(validate_assignment=True), From 84a92e848feefa9b13c6a1b3600d71496791459d Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 11:45:17 +0000 Subject: [PATCH 16/29] move decorator to extension file --- .../core/openbb_core/app/model/extension.py | 75 ++++++++++++++++++- .../core/openbb_core/app/static/container.py | 2 + .../core/openbb_core/app/static/decorators.py | 71 ------------------ 3 files changed, 76 insertions(+), 72 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 077d10b0858f..ef122a38f8bf 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -1,4 +1,5 @@ -from typing import List, Optional +import warnings +from typing import Callable, List, Optional class Extension: @@ -25,3 +26,75 @@ def __init__( self.required_credentials = [] for rq in required_credentials: self.required_credentials.append(f"{self.name.lower()}_{rq}") + + +class CachedAccessor: + """CachedAccessor""" + + def __init__(self, name: str, accessor) -> None: + self._name = name + self._accessor = accessor + + def __get__(self, obj, cls): + if obj is None: + return self._accessor + accessor_obj = self._accessor(obj) + object.__setattr__(obj, self._name, accessor_obj) + return accessor_obj + + +def register_accessor(name, cls) -> Callable: + """Register a custom accessor""" + + def decorator(accessor): + # Here we need to prevent the user from using provider names as accessor names + + if hasattr(cls, name): + warnings.warn( + f"registration of accessor '{repr(accessor)}' under name " + f"'{repr(name)}' for type '{repr(cls)}' is overriding a preexisting " + f"attribute with the same name.", + UserWarning, + ) + setattr(cls, name, CachedAccessor(name, accessor)) + # pylint: disable=protected-access + cls._accessors.add(name) + return accessor + + return decorator + + +def extend_obbject(name: str) -> Callable: + """Extend an OBBject, inspired by pandas. + + Set the following as entry_point in your extension .toml file and install it: + [tool.poetry.plugins."openbb_obbject_extension"] + example = "openbb_example:entry_point" + + Extension code: + ```python + from openbb_core.app.model.extension import Extension, extend_obbject + + entry_point = Extension(name="example", required_credentials=["api_key"]) + + @extend_obbject(name="example") + class Example: + def __init__(self, obbject): + self._obbject = obbject + + def hello(self): + api_key = self._obbject._credentials.example_api_key + print(f"Hello, this is my credential: {api_key}!") + ``` + + Usage: + >>> from openbb import obb + >>> obbject = obb.stock.load("AAPL") + >>> obbject.example.hello() + Hello, this is my credential: None! + """ + # pylint: disable=import-outside-toplevel + # Avoid circular imports + from openbb_core.app.model.obbject import OBBject + + return register_accessor(name, OBBject) diff --git a/openbb_platform/platform/core/openbb_core/app/static/container.py b/openbb_platform/platform/core/openbb_core/app/static/container.py index c69bbba13054..36b9ce8596a9 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/container.py +++ b/openbb_platform/platform/core/openbb_core/app/static/container.py @@ -12,6 +12,8 @@ class Container: def __init__(self, command_runner: CommandRunner) -> None: self._command_runner = command_runner + + # We can filter out provider credentials here, from provider names OBBject._credentials = command_runner.user_settings.credentials def _run(self, *args, **kwargs) -> Union[OBBject, pd.DataFrame, dict]: diff --git a/openbb_platform/platform/core/openbb_core/app/static/decorators.py b/openbb_platform/platform/core/openbb_core/app/static/decorators.py index e5b5f57a6b60..8d3798647241 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/decorators.py +++ b/openbb_platform/platform/core/openbb_core/app/static/decorators.py @@ -1,4 +1,3 @@ -import warnings from functools import wraps from typing import Any, Callable, Optional, TypeVar, overload @@ -35,73 +34,3 @@ def wrapper(*f_args, **f_kwargs): return wrapper return decorated if func is None else decorated(func) - - -class CachedAccessor: - """CachedAccessor""" - - def __init__(self, name: str, accessor) -> None: - self._name = name - self._accessor = accessor - - def __get__(self, obj, cls): - if obj is None: - return self._accessor - accessor_obj = self._accessor(obj) - object.__setattr__(obj, self._name, accessor_obj) - return accessor_obj - - -def register_accessor(name, cls) -> Callable: - """Register a custom accessor""" - - def decorator(accessor): - if hasattr(cls, name): - warnings.warn( - f"registration of accessor '{repr(accessor)}' under name " - f"'{repr(name)}' for type '{repr(cls)}' is overriding a preexisting " - f"attribute with the same name.", - UserWarning, - ) - setattr(cls, name, CachedAccessor(name, accessor)) - # pylint: disable=protected-access - cls._accessors.add(name) - return accessor - - return decorator - - -def extend_obbject(name: str) -> Callable: - """Extend an OBBject, inspired by pandas. - - Set the following as entry_point in your extension .toml file and install it: - [tool.poetry.plugins."openbb_obbject_extension"] - example = "openbb_example:entry_point" - - Extension code: - ```python - from openbb_core.app.model.extension import Extension - from openbb_core.app.static.decorators import extend_obbject - - entry_point = Extension(name="example", required_credentials=["api_key"]) - @extend_obbject(name="example") - class Example: - def __init__(self, obbject): - self._obbject = obbject - - def hello(self): - api_key = self._obbject._credentials.example_api_key - print(f"Hello, this is my credential: {api_key}!") - ``` - - Usage: - >>> from openbb import obb - >>> obbject = obb.stock.load("AAPL") - >>> obbject.example.hello() - Hello, this is my credential: None! - """ - # pylint: disable=import-outside-toplevel - # Avoid circular imports - from openbb_core.app.model.obbject import OBBject - - return register_accessor(name, OBBject) From f6657616b6c9cff43e0d522696dc84ee19d380a9 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 12:04:50 +0000 Subject: [PATCH 17/29] rename method extend_obbject --- .../core/openbb_core/app/model/extension.py | 115 +++++++++--------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index ef122a38f8bf..3e7bc0f99975 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -27,6 +27,64 @@ def __init__( for rq in required_credentials: self.required_credentials.append(f"{self.name.lower()}_{rq}") + @property + def accessor(self) -> Callable: + """Extend an OBBject, inspired by pandas. + + Set the following as entry_point in your extension .toml file and install it: + [tool.poetry.plugins."openbb_obbject_extension"] + example = "openbb_example:entry_point" + + Extension code: + ```python + from openbb_core.app.model.extension import Extension, extend_obbject + + entry_point = Extension(name="example", required_credentials=["api_key"]) + + @entry_point.accessor + class Example: + def __init__(self, obbject): + self._obbject = obbject + + def hello(self): + api_key = self._obbject._credentials.example_api_key + print(f"Hello, this is my credential: {api_key}!") + ``` + + Usage: + >>> from openbb import obb + >>> obbject = obb.stock.load("AAPL") + >>> obbject.example.hello() + Hello, this is my credential: None! + """ + # pylint: disable=import-outside-toplevel + # Avoid circular imports + + from openbb_core.app.model.obbject import OBBject + + return self.register_accessor(self.name, OBBject) + + @staticmethod + def register_accessor(name, cls) -> Callable: + """Register a custom accessor""" + + def decorator(accessor): + # Here we need to prevent the user from using provider names as accessor names + + if hasattr(cls, name): + warnings.warn( + f"registration of accessor '{repr(accessor)}' under name " + f"'{repr(name)}' for type '{repr(cls)}' is overriding a preexisting " + f"attribute with the same name.", + UserWarning, + ) + setattr(cls, name, CachedAccessor(name, accessor)) + # pylint: disable=protected-access + cls._accessors.add(name) + return accessor + + return decorator + class CachedAccessor: """CachedAccessor""" @@ -41,60 +99,3 @@ def __get__(self, obj, cls): accessor_obj = self._accessor(obj) object.__setattr__(obj, self._name, accessor_obj) return accessor_obj - - -def register_accessor(name, cls) -> Callable: - """Register a custom accessor""" - - def decorator(accessor): - # Here we need to prevent the user from using provider names as accessor names - - if hasattr(cls, name): - warnings.warn( - f"registration of accessor '{repr(accessor)}' under name " - f"'{repr(name)}' for type '{repr(cls)}' is overriding a preexisting " - f"attribute with the same name.", - UserWarning, - ) - setattr(cls, name, CachedAccessor(name, accessor)) - # pylint: disable=protected-access - cls._accessors.add(name) - return accessor - - return decorator - - -def extend_obbject(name: str) -> Callable: - """Extend an OBBject, inspired by pandas. - - Set the following as entry_point in your extension .toml file and install it: - [tool.poetry.plugins."openbb_obbject_extension"] - example = "openbb_example:entry_point" - - Extension code: - ```python - from openbb_core.app.model.extension import Extension, extend_obbject - - entry_point = Extension(name="example", required_credentials=["api_key"]) - - @extend_obbject(name="example") - class Example: - def __init__(self, obbject): - self._obbject = obbject - - def hello(self): - api_key = self._obbject._credentials.example_api_key - print(f"Hello, this is my credential: {api_key}!") - ``` - - Usage: - >>> from openbb import obb - >>> obbject = obb.stock.load("AAPL") - >>> obbject.example.hello() - Hello, this is my credential: None! - """ - # pylint: disable=import-outside-toplevel - # Avoid circular imports - from openbb_core.app.model.obbject import OBBject - - return register_accessor(name, OBBject) From a68c8701808c881558f6919cf0e88e9912c406a4 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:27:52 +0000 Subject: [PATCH 18/29] changes in creds model --- .../core/openbb_core/app/model/credentials.py | 48 +++++++++++-------- .../core/openbb_core/app/model/obbject.py | 2 +- .../core/openbb_core/app/static/container.py | 2 - 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index ef03f69ea63a..057eafe255c4 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -1,14 +1,16 @@ import traceback -from typing import Any, Dict, Optional, Set, Tuple +from typing import Dict, Optional, Set, Tuple from importlib_metadata import entry_points from pydantic import ( BaseModel, ConfigDict, + Field, SecretStr, create_model, - model_serializer, ) +from pydantic.functional_serializers import PlainSerializer +from typing_extensions import Annotated from openbb_core.app.model.extension import Extension from openbb_core.app.provider_interface import ProviderInterface @@ -18,48 +20,60 @@ class LoadingError(Exception): """Error loading extension.""" +# @model_serializer blocks model_dump with pydantic parameters (include, exclude) +OBBSecretStr = Annotated[ + SecretStr, + PlainSerializer( + lambda x: x.get_secret_value(), return_type=str, when_used="json-unless-none" + ), +] + + class CredentialsLoader: """Here we create the Credentials model""" - credentials: Set[str] = set() + credentials: Dict[str, Set[str]] = {} @staticmethod def prepare( - required_credentials: Set[str], + required_credentials: Dict[str, Set[str]], ) -> Dict[str, Tuple[object, None]]: """Prepare credentials map to be used in the Credentials model""" formatted: Dict[str, Tuple[object, None]] = {} - for c in required_credentials: - formatted[c] = (Optional[SecretStr], None) + for origin, creds in required_credentials.items(): + for c in creds: + formatted[c] = ( + Optional[OBBSecretStr], + Field( + default=None, description=origin + ), # register the credential origin (obbject, providers) + ) return formatted def from_obbject(self) -> None: """Load credentials from OBBject extensions""" + self.credentials["obbject"] = set() for entry_point in sorted(entry_points(group="openbb_obbject_extension")): try: entry = entry_point.load() if isinstance(entry, Extension): for c in entry.required_credentials: - self.credentials.add(c) + self.credentials["obbject"].add(c) except Exception as e: traceback.print_exception(type(e), e, e.__traceback__) raise LoadingError(f"Invalid extension '{entry_point.name}'") from e def from_providers(self) -> None: """Load credentials from providers""" + self.credentials["providers"] = set() for c in ProviderInterface().required_credentials: - self.credentials.add(c) - - def from_routers(self) -> None: - """Load credentials from routers""" - # Placeholder for future use if needed + self.credentials["providers"].add(c) def load(self) -> BaseModel: """Load credentials from providers""" self.from_obbject() self.from_providers() - self.from_routers() return create_model( # type: ignore "Credentials", __config__=ConfigDict(validate_assignment=True), @@ -73,14 +87,6 @@ def load(self) -> BaseModel: class Credentials(_Credentials): # type: ignore """Credentials model used to store provider credentials""" - @model_serializer(when_used="json-unless-none") - def _serialize(self) -> Dict[str, Any]: - """Serialize credentials to a dict""" - return { - k: v.get_secret_value() if isinstance(v, SecretStr) else v - for k, v in self.__dict__.items() - } - def __repr__(self) -> str: """String representation of the credentials""" return ( diff --git a/openbb_platform/platform/core/openbb_core/app/model/obbject.py b/openbb_platform/platform/core/openbb_core/app/model/obbject.py index 2917b6fb53a6..7d1b25bea234 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/obbject.py +++ b/openbb_platform/platform/core/openbb_core/app/model/obbject.py @@ -57,7 +57,7 @@ class OBBject(Tagged, Generic[T]): default_factory=dict, description="Extra info.", ) - _credentials: BaseModel + _credentials: ClassVar[Optional[BaseModel]] = None _accessors: ClassVar[Set[str]] = set() def __repr__(self) -> str: diff --git a/openbb_platform/platform/core/openbb_core/app/static/container.py b/openbb_platform/platform/core/openbb_core/app/static/container.py index 36b9ce8596a9..c69bbba13054 100644 --- a/openbb_platform/platform/core/openbb_core/app/static/container.py +++ b/openbb_platform/platform/core/openbb_core/app/static/container.py @@ -12,8 +12,6 @@ class Container: def __init__(self, command_runner: CommandRunner) -> None: self._command_runner = command_runner - - # We can filter out provider credentials here, from provider names OBBject._credentials = command_runner.user_settings.credentials def _run(self, *args, **kwargs) -> Union[OBBject, pd.DataFrame, dict]: From 1b3be48e25c1a2d0ceeea48dcbb283f279660333 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:41:38 +0000 Subject: [PATCH 19/29] avoid credential racing --- .../platform/core/openbb_core/app/model/credentials.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index 057eafe255c4..7e896b775552 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -42,6 +42,8 @@ def prepare( formatted: Dict[str, Tuple[object, None]] = {} for origin, creds in required_credentials.items(): for c in creds: + if c in formatted: + raise ValueError(f"Credential '{c}' already in use.") formatted[c] = ( Optional[OBBSecretStr], Field( @@ -72,8 +74,9 @@ def from_providers(self) -> None: def load(self) -> BaseModel: """Load credentials from providers""" - self.from_obbject() + # We load providers first to give them priority choosing credential names self.from_providers() + self.from_obbject() return create_model( # type: ignore "Credentials", __config__=ConfigDict(validate_assignment=True), From bd0da75a6bcb9bd7960475a23c2a79bb7b42c409 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:46:46 +0000 Subject: [PATCH 20/29] doc --- .../platform/core/openbb_core/app/model/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 3e7bc0f99975..b41262f97447 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -37,7 +37,7 @@ def accessor(self) -> Callable: Extension code: ```python - from openbb_core.app.model.extension import Extension, extend_obbject + from openbb_core.app.model.extension import Extension entry_point = Extension(name="example", required_credentials=["api_key"]) From 7813fd4d91141474bf2314d8dda3cfbb6d7000a4 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:48:49 +0000 Subject: [PATCH 21/29] rename prop --- .../platform/core/openbb_core/app/model/extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index b41262f97447..254eea4bcd41 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -28,7 +28,7 @@ def __init__( self.required_credentials.append(f"{self.name.lower()}_{rq}") @property - def accessor(self) -> Callable: + def obbject_accessor(self) -> Callable: """Extend an OBBject, inspired by pandas. Set the following as entry_point in your extension .toml file and install it: @@ -41,7 +41,7 @@ def accessor(self) -> Callable: entry_point = Extension(name="example", required_credentials=["api_key"]) - @entry_point.accessor + @entry_point.obbject_accessor class Example: def __init__(self, obbject): self._obbject = obbject From 437c99321bfeb6339651adb74e988b993c0bffdf Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:50:06 +0000 Subject: [PATCH 22/29] doc --- .../platform/core/openbb_core/app/model/extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 254eea4bcd41..5d43409561a9 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -39,9 +39,9 @@ def obbject_accessor(self) -> Callable: ```python from openbb_core.app.model.extension import Extension - entry_point = Extension(name="example", required_credentials=["api_key"]) + ext = Extension(name="example", required_credentials=["api_key"]) - @entry_point.obbject_accessor + @ext.obbject_accessor class Example: def __init__(self, obbject): self._obbject = obbject From c3f2968273324216224b24bcbecf220dfa55e83b Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:50:55 +0000 Subject: [PATCH 23/29] doc --- .../platform/core/openbb_core/app/model/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 5d43409561a9..68cb3704737d 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -33,7 +33,7 @@ def obbject_accessor(self) -> Callable: Set the following as entry_point in your extension .toml file and install it: [tool.poetry.plugins."openbb_obbject_extension"] - example = "openbb_example:entry_point" + example = "openbb_example:ext" Extension code: ```python From 3d68fb358551013df068108544cd6576098c3911 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 13:57:14 +0000 Subject: [PATCH 24/29] remove comment --- .../platform/core/openbb_core/app/model/extension.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 68cb3704737d..528999ee7b2d 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -69,8 +69,6 @@ def register_accessor(name, cls) -> Callable: """Register a custom accessor""" def decorator(accessor): - # Here we need to prevent the user from using provider names as accessor names - if hasattr(cls, name): warnings.warn( f"registration of accessor '{repr(accessor)}' under name " From 75ef2a7f7ce11af6e5c4a60e232f401021715d6c Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 14:02:59 +0000 Subject: [PATCH 25/29] comment some code --- .../platform/core/openbb_core/app/model/credentials.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/credentials.py b/openbb_platform/platform/core/openbb_core/app/model/credentials.py index 7e896b775552..79181ef716ab 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/credentials.py +++ b/openbb_platform/platform/core/openbb_core/app/model/credentials.py @@ -42,8 +42,9 @@ def prepare( formatted: Dict[str, Tuple[object, None]] = {} for origin, creds in required_credentials.items(): for c in creds: - if c in formatted: - raise ValueError(f"Credential '{c}' already in use.") + # Not sure we should do this, if you require the same credential it breaks + # if c in formatted: + # raise ValueError(f"Credential '{c}' already in use.") formatted[c] = ( Optional[OBBSecretStr], Field( From 9a5a1965b834a4e415914522af4db58b0f25fa92 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 15:27:19 +0000 Subject: [PATCH 26/29] free extension names --- .../core/openbb_core/app/model/extension.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 528999ee7b2d..4d434f479c49 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -15,17 +15,12 @@ def __init__( Parameters ---------- name : str - Name of the provider. + Name of the extension. required_credentials : Optional[List[str]], optional List of required credentials, by default None """ self.name = name - if required_credentials is None: - self.required_credentials: List = [] - else: - self.required_credentials = [] - for rq in required_credentials: - self.required_credentials.append(f"{self.name.lower()}_{rq}") + self.required_credentials = required_credentials or [] @property def obbject_accessor(self) -> Callable: @@ -39,7 +34,7 @@ def obbject_accessor(self) -> Callable: ```python from openbb_core.app.model.extension import Extension - ext = Extension(name="example", required_credentials=["api_key"]) + ext = Extension(name="example", required_credentials=["some_api_key"]) @ext.obbject_accessor class Example: @@ -47,7 +42,7 @@ def __init__(self, obbject): self._obbject = obbject def hello(self): - api_key = self._obbject._credentials.example_api_key + api_key = self._obbject._credentials.some_api_key print(f"Hello, this is my credential: {api_key}!") ``` From 0cbdbb557aad93ddfaf98247f05d93fbd02e7d08 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 15:58:35 +0000 Subject: [PATCH 27/29] docstring --- .../core/openbb_core/app/model/extension.py | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 4d434f479c49..5e35d67477c7 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -3,7 +3,40 @@ class Extension: - """Serves as extension entry point and must be created by each extension package.""" + """Serves as extension entry point and must be created by each extension package. + + Steps to create an extension: + + 1. Set the following as entry_point in your extension .toml file and install it: + ```toml + [tool.poetry.plugins."openbb_obbject_extension"] + example = "openbb_example:ext" + ``` + + 2. Extension code: + ```python + from openbb_core.app.model.extension import Extension + ext = Extension(name="example", required_credentials=["some_api_key"]) + ``` + + 3. Optionally declare an obbject accessor: + ```python + @ext.obbject_accessor + class Example: + def __init__(self, obbject): + self._obbject = obbject + + def hello(self): + api_key = self._obbject._credentials.some_api_key + print(f"Hello, this is my credential: {api_key}!") + ``` + + Usage: + >>> from openbb import obb + >>> obbject = obb.stock.load("AAPL") + >>> obbject.example.hello() + Hello, this is my credential: None! + """ def __init__( self, @@ -24,34 +57,7 @@ def __init__( @property def obbject_accessor(self) -> Callable: - """Extend an OBBject, inspired by pandas. - - Set the following as entry_point in your extension .toml file and install it: - [tool.poetry.plugins."openbb_obbject_extension"] - example = "openbb_example:ext" - - Extension code: - ```python - from openbb_core.app.model.extension import Extension - - ext = Extension(name="example", required_credentials=["some_api_key"]) - - @ext.obbject_accessor - class Example: - def __init__(self, obbject): - self._obbject = obbject - - def hello(self): - api_key = self._obbject._credentials.some_api_key - print(f"Hello, this is my credential: {api_key}!") - ``` - - Usage: - >>> from openbb import obb - >>> obbject = obb.stock.load("AAPL") - >>> obbject.example.hello() - Hello, this is my credential: None! - """ + """Extend an OBBject, inspired by pandas.""" # pylint: disable=import-outside-toplevel # Avoid circular imports From 5ff9ab7e70770509f261596a28e67f986fb0d115 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 15:59:29 +0000 Subject: [PATCH 28/29] doc --- .../platform/core/openbb_core/app/model/extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index 5e35d67477c7..dac3b919143b 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -19,7 +19,7 @@ class Extension: ext = Extension(name="example", required_credentials=["some_api_key"]) ``` - 3. Optionally declare an obbject accessor: + 3. Optionally declare an obbject accessor, it will use the extension name: ```python @ext.obbject_accessor class Example: From 41d0fa09590cb9c4a76248606b505933bbe8fa58 Mon Sep 17 00:00:00 2001 From: Diogo Sousa Date: Mon, 30 Oct 2023 18:54:20 +0000 Subject: [PATCH 29/29] docs --- openbb_platform/platform/core/README.md | 72 ++++++++++++++----- .../core/openbb_core/app/model/extension.py | 32 +-------- 2 files changed, 57 insertions(+), 47 deletions(-) diff --git a/openbb_platform/platform/core/README.md b/openbb_platform/platform/core/README.md index e81d046ed330..19a2e4557eaa 100644 --- a/openbb_platform/platform/core/README.md +++ b/openbb_platform/platform/core/README.md @@ -14,11 +14,11 @@ - [4.1 Static version](#41-static-version) - [4.1.1. OBBject](#411-obbject) - [Helpers](#helpers) + - [Extensions](#extensions) - [4.1.2. Utilities](#412-utilities) - [User settings](#user-settings) + - [Preferences](#preferences) - [System settings](#system-settings) - - [Preferences](#preferences) - - [Available preferences and its descriptions](#available-preferences-and-its-descriptions) - [Coverage](#coverage) - [4.1.3. OpenBB Hub Account](#413-openbb-hub-account) - [4.1.4. Command execution](#414-command-execution) @@ -63,7 +63,7 @@ poetry install Build a Python package: ```bash -poetry new openbb-sdk-my_extension +poetry new openbb-platform-my_extension ``` ### Command @@ -235,11 +235,52 @@ date } ``` +#### Extensions + +Steps to create an `OBBject` extension: + +1. Set the following as entry point in your extension .toml file and install it: + + ```toml + ... + [tool.poetry.plugins."openbb_obbject_extension"] + example = "openbb_example:ext" + ``` + +2. Extension code: + + ```python + from openbb_core.app.model.extension import Extension + ext = Extension(name="example", required_credentials=["some_api_key"]) + ``` + +3. Optionally declare an `OBBject` accessor, it will use the extension name: + + ```python + @ext.obbject_accessor + class Example: + def __init__(self, obbject): + self._obbject = obbject + + def hello(self): + api_key = self._obbject._credentials.some_api_key + print(f"Hello, this is my credential: {api_key}!") + ``` + + Usage: + + ```shell + >>> from openbb import obb + >>> obbject = obb.stock.load("AAPL") + >>> obbject.example.hello() + Hello, this is my credential: None! + ``` + ### 4.1.2. Utilities #### User settings -These are your user settings, you can change them anytime and they will be applied. Don't forget to `sdk.account.save()` if you want these changes to persist. +These are your user settings, you can change them anytime and they will be applied. Don't forget to `obb.account.save()` if you want these changes to persist. ```python from openbb import obb @@ -250,16 +291,6 @@ obb.user.preferences obb.user.defaults ``` -#### System settings - -Check your system settings. - -```python -from openbb import obb - -obb.system -``` - #### Preferences Check your preferences by adjusting the `user_settings.json` file inside your **home** directory. @@ -282,7 +313,7 @@ Here is an example of how your `user_settings.json` file can look like: > Note that the user preferences shouldn't be confused with environment variables. -##### Available preferences and its descriptions +These are the available preferences and respective descriptions: |Preference |Default |Description | |---------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -300,6 +331,15 @@ Here is an example of how your `user_settings.json` file can look like: |metadata |True |Enables or disables the collection of metadata which provides information about operations including arguments duration route and timestamp. Disabling this feature may improve performance in cases where contextual information is not needed or when the additional computation time and storage space are a concern.| |output_type |OBBject |Specifies the type of data the application will output when a command or endpoint is accessed. Note that choosing data formats only available in Python such as `dataframe`, `numpy` or `polars` will render the application's API non-functional. | +#### System settings + +Check your system settings. + +```python +from openbb import obb + +obb.system +``` #### Coverage @@ -389,7 +429,7 @@ To apply an environment variable use one of the following: ```python import os os.environ["OPENBB_DEBUG_MODE"] = "True" - from openbb import sdk + from openbb import obb ``` 2. Persistent: create a `.env` file in `/.openbb_platform` folder inside your home directory with diff --git a/openbb_platform/platform/core/openbb_core/app/model/extension.py b/openbb_platform/platform/core/openbb_core/app/model/extension.py index dac3b919143b..587e337c8d00 100644 --- a/openbb_platform/platform/core/openbb_core/app/model/extension.py +++ b/openbb_platform/platform/core/openbb_core/app/model/extension.py @@ -5,37 +5,7 @@ class Extension: """Serves as extension entry point and must be created by each extension package. - Steps to create an extension: - - 1. Set the following as entry_point in your extension .toml file and install it: - ```toml - [tool.poetry.plugins."openbb_obbject_extension"] - example = "openbb_example:ext" - ``` - - 2. Extension code: - ```python - from openbb_core.app.model.extension import Extension - ext = Extension(name="example", required_credentials=["some_api_key"]) - ``` - - 3. Optionally declare an obbject accessor, it will use the extension name: - ```python - @ext.obbject_accessor - class Example: - def __init__(self, obbject): - self._obbject = obbject - - def hello(self): - api_key = self._obbject._credentials.some_api_key - print(f"Hello, this is my credential: {api_key}!") - ``` - - Usage: - >>> from openbb import obb - >>> obbject = obb.stock.load("AAPL") - >>> obbject.example.hello() - Hello, this is my credential: None! + See README.md for more information on how to create an extension. """ def __init__(