Merge branch 'develop' into bugfix/pywry-plot-paperbg

assets/README.md

@@ -0,0 +1,5 @@
+# Assets
+
+This folder should hold assets read by OpenBB applications, such as OpenBB Hub or marketing website.
+
+The goal is to be more explicit about which assets are being used externally and cannot be deleted before checking where they are used.

assets/extensions/obbject.json

@@ -0,0 +1,7 @@
+[
+    {
+        "packageName": "openbb-charting",
+        "optional": true,
+        "description": "Create custom charts from OBBject data."
+    }
+]

assets/extensions/router.json

@@ -0,0 +1,72 @@
+[
+    {
+        "packageName": "openbb-commodity",
+        "optional": false,
+        "description": "Commodity market data."
+    },
+    {
+        "packageName": "openbb-crypto",
+        "optional": false,
+        "description": "Cryptocurrency market data."
+    },
+    {
+        "packageName": "openbb-currency",
+        "optional": false,
+        "description": "Foreign exchange (FX) market data."
+    },
+    {
+        "packageName": "openbb-derivatives",
+        "optional": false,
+        "description": "Derivatives market data."
+    },
+    {
+        "packageName": "openbb-econometrics",
+        "optional": true,
+        "description": "Econometrics analysis tools."
+    },
+    {
+        "packageName": "openbb-economy",
+        "optional": false,
+        "description": "Economic data."
+    },
+    {
+        "packageName": "openbb-equity",
+        "optional": false,
+        "description": "Equity market data."
+    },
+    {
+        "packageName": "openbb-etf",
+        "optional": false,
+        "description": "Exchange Traded Funds market data."
+    },
+    {
+        "packageName": "openbb-fixedincome",
+        "optional": false,
+        "description": "Fixed Income market data."
+    },
+    {
+        "packageName": "openbb-index",
+        "optional": false,
+        "description": "Indices data."
+    },
+    {
+        "packageName": "openbb-news",
+        "optional": false,
+        "description": "Financial market news data."
+    },
+    {
+        "packageName": "openbb-quantitative",
+        "optional": true,
+        "description": "Quantitative analysis tools."
+    },
+    {
+        "packageName": "openbb-regulators",
+        "optional": false,
+        "description": "Financial market regulators data."
+    },
+    {
+        "packageName": "openbb-technical",
+        "optional": true,
+        "description": "Technical Analysis tools."
+    }
+]

assets/scripts/generate_extension_data.py

@@ -0,0 +1,122 @@
+"""Generate assets from modules."""
+
+from importlib import import_module
+from json import dump
+from pathlib import Path
+from typing import Any, Dict, List
+
+from poetry.core.pyproject.toml import PyProjectTOML
+
+THIS_DIR = Path(__file__).parent
+OPENBB_PLATFORM_PATH = Path(THIS_DIR, "..", "..", "openbb_platform")
+PROVIDERS_PATH = OPENBB_PLATFORM_PATH / "providers"
+EXTENSIONS_PATH = OPENBB_PLATFORM_PATH / "extensions"
+OBBJECT_EXTENSIONS_PATH = OPENBB_PLATFORM_PATH / "obbject_extensions"
+
+OPENBB_PLATFORM_TOML = PyProjectTOML(OPENBB_PLATFORM_PATH / "pyproject.toml")
+
+
+def to_title(string: str) -> str:
+    """Format string to title."""
+    return " ".join(string.split("_")).title()
+
+
+def get_packages(path: Path, plugin_key: str) -> Dict[str, Any]:
+    """Get packages."""
+    SKIP = ["tests", "__pycache__"]
+    folders = [f for f in path.glob("*") if f.is_dir() and f.stem not in SKIP]
+    packages: Dict[str, Any] = {}
+    for f in folders:
+        pyproject = PyProjectTOML(Path(f, "pyproject.toml"))
+        poetry = pyproject.data["tool"]["poetry"]
+        name = poetry["name"]
+        plugin = poetry.get("plugins", {}).get(plugin_key)
+        packages[name] = {"plugin": list(plugin.values())[0] if plugin else ""}
+    return packages
+
+
+def write(filename: str, data: Any):
+    """Write to json."""
+    with open(Path(THIS_DIR, "..", "extensions", f"{filename}.json"), "w") as json_file:
+        dump(data, json_file, indent=4)
+
+
+def to_camel(string: str):
+    """Convert string to camel case."""
+    components = string.split("_")
+    return components[0] + "".join(x.title() for x in components[1:])
+
+
+def createItem(package_name: str, obj: object, obj_attrs: List[str]) -> Dict[str, Any]:
+    """Create dictionary item from object attributes."""
+    pkg_spec = OPENBB_PLATFORM_TOML.data["tool"]["poetry"]["dependencies"].get(
+        package_name
+    )
+    optional = pkg_spec.get("optional", False) if isinstance(pkg_spec, dict) else False
+    item = {"packageName": package_name, "optional": optional}
+    item.update(
+        {to_camel(a): getattr(obj, a) for a in obj_attrs if getattr(obj, a) is not None}
+    )
+    return item
+
+
+def generate_provider_extensions() -> None:
+    """Generate providers_extensions.json."""
+    packages = get_packages(PROVIDERS_PATH, "openbb_provider_extension")
+    data: List[Dict[str, Any]] = []
+    obj_attrs = [
+        "repr_name",
+        "description",
+        "credentials",
+        "v3_credentials",
+        "website",
+        "instructions",
+    ]
+
+    for pkg_name, details in sorted(packages.items()):
+        plugin = details.get("plugin", "")
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            provider_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, provider_obj, obj_attrs))
+    write("provider", data)
+
+
+def generate_router_extensions() -> None:
+    """Generate router_extensions.json."""
+    packages = get_packages(EXTENSIONS_PATH, "openbb_core_extension")
+    data: List[Dict[str, Any]] = []
+    obj_attrs = ["description"]
+    for pkg_name, details in sorted(packages.items()):
+        plugin = details.get("plugin", "")
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            router_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, router_obj, obj_attrs))
+    write("router", data)
+
+
+def generate_obbject_extensions() -> None:
+    """Generate obbject_extensions.json."""
+    packages = get_packages(OBBJECT_EXTENSIONS_PATH, "openbb_obbject_extension")
+    data: List[Dict[str, Any]] = []
+    obj_attrs = ["description"]
+    for pkg_name, details in sorted(packages.items()):
+        plugin = details.get("plugin", "")
+        file_obj = plugin.split(":")
+        if len(file_obj) == 2:
+            file, obj = file_obj[0], file_obj[1]
+            module = import_module(file)
+            ext_obj = getattr(module, obj)
+            data.append(createItem(pkg_name, ext_obj, obj_attrs))
+    write("obbject", data)
+
+
+if __name__ == "__main__":
+    generate_provider_extensions()
+    generate_router_extensions()
+    generate_obbject_extensions()

build/pypi/openbb_platform/PUBLISH.md

@@ -55,6 +55,7 @@
 
 1. Install the packages on Google Colaboratory via PyPi and test to check if everything is working as expected.
 2. Install the packages in a new environment locally via PyPi and test to check if everything is working as expected.
-3. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch.
-4. Merge the `release/<package>-<version>` branch to the `develop` branch.
-5. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly.
+3. Regenerate assets for external use by running `python assets/scripts/generate_extension_data.py`
+4. Open a new PR with the `release/<package>-<version>` branch pointing to the `develop` branch.
+5. Merge the `release/<package>-<version>` branch to the `develop` branch.
+6. If any bugs are encountered, create a new branch - `hotfix` for `main` and `bugfix` for `develop` and merge them accordingly.