[Feature] Replace Nasdaq SP500 Multiples With Direct Source (#6609)

* replace nasdaq sp500 multiples with direct source

* black

* filter for start/end date

* linting

* didn't commit the changes

* docstring

* black

* docstring

* unused argument

* ruff

* markdown in description

* UPPER

---------

Co-authored-by: Igor Radovanovic <[email protected]>

openbb_platform/core/openbb_core/provider/standard_models/sp500_multiples.py

@@ -1,7 +1,7 @@
 """SP500 Multiples Standard Model."""
 
 from datetime import date as dateType
-from typing import Literal, Optional
+from typing import Literal, Optional, Union
 
 from pydantic import Field
 
@@ -12,7 +12,7 @@
     QUERY_DESCRIPTIONS,
 )
 
-SERIES_NAMES = Literal[
+SERIES_NAME = Literal[
     "shiller_pe_month",
     "shiller_pe_year",
     "pe_year",
@@ -55,7 +55,7 @@
 class SP500MultiplesQueryParams(QueryParams):
     """SP500 Multiples Query."""
 
-    series_name: SERIES_NAMES = Field(
+    series_name: Union[SERIES_NAME, str] = Field(
         description="The name of the series. Defaults to 'pe_month'.",
         default="pe_month",
     )
@@ -71,3 +71,9 @@ class SP500MultiplesData(Data):
     """SP500 Multiples Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
+    name: str = Field(
+        description="Name of the series.",
+    )
+    value: Union[int, float] = Field(
+        description="Value of the series.",
+    )

openbb_platform/dev_install.py

@@ -1,5 +1,7 @@
 """Install for development script."""
 
+# flake8: noqa: S603
+
 import subprocess
 import sys
 from pathlib import Path
@@ -52,6 +54,7 @@
 openbb-finra = { path = "./providers/finra", optional = true, develop = true }
 openbb-finviz = { path = "./providers/finviz", optional = true, develop = true }
 openbb-government-us = { path = "./providers/government_us", optional = true, develop = true }
+openbb-multpl = { path = "./providers/multpl", optional = true, develop = true }
 openbb-nasdaq = { path = "./providers/nasdaq", optional = true, develop = true }
 openbb-seeking-alpha = { path = "./providers/seeking_alpha", optional = true, develop = true }
 openbb-stockgrid = { path = "./providers/stockgrid" , optional = true,  develop = true }
@@ -138,10 +141,14 @@ def install_platform_local(_extras: bool = False):
         extras_args = ["-E", "all"] if _extras else []
 
         subprocess.run(
-            CMD + ["lock", "--no-update"], cwd=PLATFORM_PATH, check=True  # noqa: S603
+            CMD + ["lock", "--no-update"],
+            cwd=PLATFORM_PATH,
+            check=True,
         )
         subprocess.run(
-            CMD + ["install"] + extras_args, cwd=PLATFORM_PATH, check=True  # noqa: S603
+            CMD + ["install"] + extras_args,
+            cwd=PLATFORM_PATH,
+            check=True,
         )
 
     except (Exception, KeyboardInterrupt) as e:
@@ -179,7 +186,9 @@ def install_platform_cli():
         CMD = [sys.executable, "-m", "poetry"]
 
         subprocess.run(
-            CMD + ["lock", "--no-update"], cwd=CLI_PATH, check=True  # noqa: S603
+            CMD + ["lock", "--no-update"],
+            cwd=CLI_PATH,
+            check=True,  # noqa: S603
         )
         subprocess.run(CMD + ["install"], cwd=CLI_PATH, check=True)  # noqa: S603
 

openbb_platform/extensions/index/integration/test_index_api.py

@@ -197,7 +197,15 @@ def test_index_snapshots(params, headers):
                 "transform": "diff",
                 "provider": "nasdaq",
             }
-        )
+        ),
+        (
+            {
+                "series_name": "pe_month",
+                "start_date": None,
+                "end_date": None,
+                "provider": "multpl",
+            }
+        ),
     ],
 )
 @pytest.mark.integration

openbb_platform/extensions/index/integration/test_index_python.py

@@ -187,6 +187,14 @@ def test_index_snapshots(params, obb):
                 "provider": "nasdaq",
             }
         ),
+        (
+            {
+                "series_name": "pe_month",
+                "start_date": None,
+                "end_date": None,
+                "provider": "multpl",
+            }
+        ),
     ],
 )
 @pytest.mark.integration

openbb_platform/extensions/index/openbb_index/index_router.py

@@ -93,8 +93,8 @@ async def search(
 @router.command(
     model="SP500Multiples",
     examples=[
-        APIEx(parameters={"provider": "nasdaq"}),
-        APIEx(parameters={"series_name": "shiller_pe_year", "provider": "nasdaq"}),
+        APIEx(parameters={"provider": "multpl"}),
+        APIEx(parameters={"series_name": "shiller_pe_year", "provider": "multpl"}),
     ],
 )
 async def sp500_multiples(

openbb_platform/providers/multpl/README.md

@@ -0,0 +1,13 @@
+# Multpl Provider Extension
+
+This is an implementation of the data published to (https://multpl.com)[https;//multpl.com]
+
+## Installation
+
+```
+pip install openbb-multpl
+```
+
+## Endpoints
+
+- `obb.index.sp500_multiples`

openbb_platform/providers/multpl/__init__.py

@@ -0,0 +1 @@
+"""Multpl Provider Extension."""

openbb_platform/providers/multpl/openbb_multpl/__init__.py

@@ -0,0 +1,13 @@
+"""Multpl Provider Module."""
+
+from openbb_core.provider.abstract.provider import Provider
+from openbb_multpl.models.sp500_multiples import MultplSP500MultiplesFetcher
+
+multpl_provider = Provider(
+    name="multpl",
+    website="https://www.multpl.com/",
+    description="""Public broad-market data published to https://multpl.com.""",
+    fetcher_dict={
+        "SP500Multiples": MultplSP500MultiplesFetcher,
+    },
+)

openbb_platform/providers/multpl/openbb_multpl/models/__init__.py

@@ -0,0 +1 @@
+"""Multpl Provider Models."""

openbb_platform/providers/multpl/openbb_multpl/models/sp500_multiples.py

@@ -0,0 +1,179 @@
+"""Multpl S&P 500 Multiples Model."""
+
+# pylint: disable=unused-argument
+
+from typing import Any, Dict, List, Optional
+from warnings import warn
+
+from openbb_core.app.model.abstract.error import OpenBBError
+from openbb_core.provider.abstract.fetcher import Fetcher
+from openbb_core.provider.standard_models.sp500_multiples import (
+    SP500MultiplesData,
+    SP500MultiplesQueryParams,
+)
+from openbb_core.provider.utils.errors import EmptyDataError
+from pydantic import field_validator
+
+BASE_URL = "https://www.multpl.com/"
+
+URL_DICT = {
+    "shiller_pe_month": "shiller-pe/table/by-month",
+    "shiller_pe_year": "shiller-pe/table/by-year",
+    "pe_year": "s-p-500-pe-ratio/table/by-year",
+    "pe_month": "s-p-500-pe-ratio/table/by-month",
+    "dividend_year": "s-p-500-dividend/table/by-year",
+    "dividend_month": "s-p-500-dividend/table/by-month",
+    "dividend_growth_quarter": "s-p-500-dividend-growth/table/by-quarter",
+    "dividend_growth_year": "s-p-500-dividend-growth/table/by-year",
+    "dividend_yield_year": "s-p-500-dividend-yield/table/by-year",
+    "dividend_yield_month": "s-p-500-dividend-yield/table/by-month",
+    "earnings_year": "s-p-500-earnings/table/by-year",
+    "earnings_month": "s-p-500-earnings/table/by-month",
+    "earnings_growth_year": "s-p-500-earnings-growth/table/by-year",
+    "earnings_growth_quarter": "s-p-500-earnings-growth/table/by-quarter",
+    "real_earnings_growth_year": "s-p-500-real-earnings-growth/table/by-year",
+    "real_earnings_growth_quarter": "s-p-500-real-earnings-growth/table/by-quarter",
+    "earnings_yield_year": "s-p-500-earnings-yield/table/by-year",
+    "earnings_yield_month": "s-p-500-earnings-yield/table/by-month",
+    "real_price_year": "s-p-500-historical-prices/table/by-year",
+    "real_price_month": "s-p-500-historical-prices/table/by-month",
+    "inflation_adjusted_price_year": "inflation-adjusted-s-p-500/table/by-year",
+    "inflation_adjusted_price_month": "inflation-adjusted-s-p-500/table/by-month",
+    "sales_year": "s-p-500-sales/table/by-year",
+    "sales_quarter": "s-p-500-sales/table/by-quarter",
+    "sales_growth_year": "s-p-500-sales-growth/table/by-year",
+    "sales_growth_quarter": "s-p-500-sales-growth/table/by-quarter",
+    "real_sales_year": "s-p-500-real-sales/table/by-year",
+    "real_sales_quarter": "s-p-500-real-sales/table/by-quarter",
+    "real_sales_growth_year": "s-p-500-real-sales-growth/table/by-year",
+    "real_sales_growth_quarter": "s-p-500-real-sales-growth/table/by-quarter",
+    "price_to_sales_year": "s-p-500-price-to-sales/table/by-year",
+    "price_to_sales_quarter": "s-p-500-price-to-sales/table/by-quarter",
+    "price_to_book_value_year": "s-p-500-price-to-book/table/by-year",
+    "price_to_book_value_quarter": "s-p-500-price-to-book/table/by-quarter",
+    "book_value_year": "s-p-500-book-value/table/by-year",
+    "book_value_quarter": "s-p-500-book-value/table/by-quarter",
+}
+
+
+class MultplSP500MultiplesQueryParams(SP500MultiplesQueryParams):
+    """Multpl S&P 500 Multiples Query Params."""
+
+    __json_schema_extra__ = {
+        "series_name": {
+            "multiple_items_allowed": True,
+            "choices": sorted(list(URL_DICT)),
+        }
+    }
+
+    @field_validator("series_name", mode="before", check_fields=False)
+    @classmethod
+    def validate_series_name(cls, v):
+        """Validate series_name."""
+        series = v.split(",")
+        new_values: List = []
+        for s in series:
+            if s not in URL_DICT:
+                raise OpenBBError(
+                    f"{s} is not a valid `series_name`. Choices are: \n{sorted(list(URL_DICT))}\n"
+                )
+            new_values.append(s)
+        if not new_values:
+            raise OpenBBError(
+                f"No valid series names provided. Choices are: \n{sorted(list(URL_DICT))}\n"
+            )
+        return ",".join(new_values)
+
+
+class MultplSP500MultiplesData(SP500MultiplesData):
+    """Multpl S&P 500 Multiples Data."""
+
+    __alias_dict__ = {
+        "date": "Date",
+        "value": "Value",
+    }
+
+
+class MultplSP500MultiplesFetcher(
+    Fetcher[
+        MultplSP500MultiplesQueryParams,
+        List[MultplSP500MultiplesData],
+    ]
+):
+    """Multpl S&P 500 Multiples Fetcher."""
+
+    @staticmethod
+    def transform_query(params: Dict[str, Any]) -> MultplSP500MultiplesQueryParams:
+        """Transform the query params."""
+        return MultplSP500MultiplesQueryParams(**params)
+
+    @staticmethod
+    async def aextract_data(
+        query: MultplSP500MultiplesQueryParams,
+        credentials: Optional[Dict[str, str]],
+        **kwargs: Any,
+    ) -> List[Dict]:
+        """Extract data."""
+        # pylint: disable=import-outside-toplevel
+        import asyncio  # noqa
+        from io import StringIO
+        from openbb_core.provider.utils.helpers import amake_request
+        from numpy import nan
+        from pandas import read_html, to_datetime
+
+        series = query.series_name.split(",")
+        urls = {s: f"{BASE_URL}{URL_DICT[s]}" for s in series}
+        results: List = []
+
+        async def response_callback(response, _):
+            """Response callback."""
+            return await response.text()
+
+        async def get_one(url, series):
+            """Get data for one series."""
+            res = await amake_request(url, response_callback=response_callback)
+            if res:
+                df = read_html(StringIO(res))[0]  # type: ignore
+                if not df.empty:
+                    df["Date"] = to_datetime(df["Date"]).dt.date
+                    df = df.sort_values("Date").reset_index(drop=True)
+                    if query.start_date:
+                        df = df[df["Date"] >= query.start_date]
+                    if query.end_date:
+                        df = df[df["Date"] <= query.end_date]
+                    df["Value"] = df["Value"].apply(
+                        lambda x: (
+                            x.strip().replace("† ", "").replace("%", "")
+                            if isinstance(x, str)
+                            else x
+                        )
+                    )
+                    df["name"] = series
+                    if "growth" in series or "yield" in series:
+                        df["Value"] = df["Value"].astype(float) / 100
+
+                    results.extend(df.replace({nan: None}).to_dict(orient="records"))
+            else:
+                warn(f"Failed to get data for {series}.")
+
+        await asyncio.gather(*[get_one(url, series) for series, url in urls.items()])
+
+        if not results:
+            raise EmptyDataError("The request was returned empty.")
+
+        return results
+
+    @staticmethod
+    def transform_data(
+        query: MultplSP500MultiplesQueryParams,
+        data: List[Dict],
+        **kwargs: Any,
+    ) -> List[MultplSP500MultiplesData]:
+        """Transform and validate the data."""
+        return [
+            MultplSP500MultiplesData.model_validate(d)
+            for d in sorted(
+                data,
+                key=lambda x: (x["Date"], x["name"]),
+            )
+        ]

openbb_platform/providers/multpl/openbb_multpl/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities and Helpers."""