Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Replace Nasdaq SP500 Multiples With Direct Source #6609

Merged
merged 15 commits into from
Aug 8, 2024
Merged
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""SP500 Multiples Standard Model."""

from datetime import date as dateType
from typing import Literal, Optional
from typing import Literal, Optional, Union

from pydantic import Field

Expand All @@ -12,7 +12,7 @@
QUERY_DESCRIPTIONS,
)

SERIES_NAMES = Literal[
SeriesNames = Literal[
deeleeramone marked this conversation as resolved.
Show resolved Hide resolved
"shiller_pe_month",
"shiller_pe_year",
"pe_year",
Expand Down Expand Up @@ -55,7 +55,7 @@
class SP500MultiplesQueryParams(QueryParams):
"""SP500 Multiples Query."""

series_name: SERIES_NAMES = Field(
series_name: Union[SeriesNames, str] = Field(
description="The name of the series. Defaults to 'pe_month'.",
default="pe_month",
)
Expand All @@ -71,3 +71,9 @@ class SP500MultiplesData(Data):
"""SP500 Multiples Data."""

date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
name: str = Field(
description="Name of the series.",
)
value: Union[int, float] = Field(
description="Value of the series.",
)
15 changes: 12 additions & 3 deletions openbb_platform/dev_install.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Install for development script."""

# flake8: noqa: S603

import subprocess
import sys
from pathlib import Path
Expand Down Expand Up @@ -52,6 +54,7 @@
openbb-finra = { path = "./providers/finra", optional = true, develop = true }
openbb-finviz = { path = "./providers/finviz", optional = true, develop = true }
openbb-government-us = { path = "./providers/government_us", optional = true, develop = true }
openbb-multpl = { path = "./providers/multpl", optional = true, develop = true }
openbb-nasdaq = { path = "./providers/nasdaq", optional = true, develop = true }
openbb-seeking-alpha = { path = "./providers/seeking_alpha", optional = true, develop = true }
openbb-stockgrid = { path = "./providers/stockgrid" , optional = true, develop = true }
Expand Down Expand Up @@ -138,10 +141,14 @@ def install_platform_local(_extras: bool = False):
extras_args = ["-E", "all"] if _extras else []

subprocess.run(
CMD + ["lock", "--no-update"], cwd=PLATFORM_PATH, check=True # noqa: S603
CMD + ["lock", "--no-update"],
cwd=PLATFORM_PATH,
check=True,
)
subprocess.run(
CMD + ["install"] + extras_args, cwd=PLATFORM_PATH, check=True # noqa: S603
CMD + ["install"] + extras_args,
cwd=PLATFORM_PATH,
check=True,
)

except (Exception, KeyboardInterrupt) as e:
Expand Down Expand Up @@ -179,7 +186,9 @@ def install_platform_cli():
CMD = [sys.executable, "-m", "poetry"]

subprocess.run(
CMD + ["lock", "--no-update"], cwd=CLI_PATH, check=True # noqa: S603
CMD + ["lock", "--no-update"],
cwd=CLI_PATH,
check=True, # noqa: S603
)
subprocess.run(CMD + ["install"], cwd=CLI_PATH, check=True) # noqa: S603

Expand Down
10 changes: 9 additions & 1 deletion openbb_platform/extensions/index/integration/test_index_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,15 @@ def test_index_snapshots(params, headers):
"transform": "diff",
"provider": "nasdaq",
}
)
),
(
{
"series_name": "pe_month",
"start_date": None,
"end_date": None,
"provider": "multpl",
}
),
],
)
@pytest.mark.integration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,14 @@ def test_index_snapshots(params, obb):
"provider": "nasdaq",
}
),
(
{
"series_name": "pe_month",
"start_date": None,
"end_date": None,
"provider": "multpl",
}
),
],
)
@pytest.mark.integration
Expand Down
4 changes: 2 additions & 2 deletions openbb_platform/extensions/index/openbb_index/index_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,8 @@ async def search(
@router.command(
model="SP500Multiples",
examples=[
APIEx(parameters={"provider": "nasdaq"}),
APIEx(parameters={"series_name": "shiller_pe_year", "provider": "nasdaq"}),
APIEx(parameters={"provider": "multpl"}),
APIEx(parameters={"series_name": "shiller_pe_year", "provider": "multpl"}),
],
)
async def sp500_multiples(
Expand Down
13 changes: 13 additions & 0 deletions openbb_platform/providers/multpl/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Multpl Provider Extension

This is an implementation of the data published to (https://multpl.com)[https;//multpl.com]

## Installation

```
pip install openbb-multpl
```

## Endpoints

- `obb.index.sp500_multiples`
1 change: 1 addition & 0 deletions openbb_platform/providers/multpl/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Multpl Provider Extension."""
13 changes: 13 additions & 0 deletions openbb_platform/providers/multpl/openbb_multpl/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
"""Multpl Provider Module."""

from openbb_core.provider.abstract.provider import Provider
from openbb_multpl.models.sp500_multiples import MultplSP500MultiplesFetcher

multpl_provider = Provider(
name="multpl",
website="https://www.multpl.com/",
description="""Public broad-market data published to https://multpl.com.""",
fetcher_dict={
"SP500Multiples": MultplSP500MultiplesFetcher,
},
)
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Multpl Provider Models."""
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
"""Multpl S&P 500 Multiples Model."""

# pylint: disable=unused-argument

from typing import Any, Dict, List, Optional
from warnings import warn

from openbb_core.app.model.abstract.error import OpenBBError
from openbb_core.provider.abstract.fetcher import Fetcher
from openbb_core.provider.standard_models.sp500_multiples import (
SP500MultiplesData,
SP500MultiplesQueryParams,
)
from openbb_core.provider.utils.errors import EmptyDataError
from pydantic import field_validator

BASE_URL = "https://www.multpl.com/"

URL_DICT = {
"shiller_pe_month": "shiller-pe/table/by-month",
"shiller_pe_year": "shiller-pe/table/by-year",
"pe_year": "s-p-500-pe-ratio/table/by-year",
"pe_month": "s-p-500-pe-ratio/table/by-month",
"dividend_year": "s-p-500-dividend/table/by-year",
"dividend_month": "s-p-500-dividend/table/by-month",
"dividend_growth_quarter": "s-p-500-dividend-growth/table/by-quarter",
"dividend_growth_year": "s-p-500-dividend-growth/table/by-year",
"dividend_yield_year": "s-p-500-dividend-yield/table/by-year",
"dividend_yield_month": "s-p-500-dividend-yield/table/by-month",
"earnings_year": "s-p-500-earnings/table/by-year",
"earnings_month": "s-p-500-earnings/table/by-month",
"earnings_growth_year": "s-p-500-earnings-growth/table/by-year",
"earnings_growth_quarter": "s-p-500-earnings-growth/table/by-quarter",
"real_earnings_growth_year": "s-p-500-real-earnings-growth/table/by-year",
"real_earnings_growth_quarter": "s-p-500-real-earnings-growth/table/by-quarter",
"earnings_yield_year": "s-p-500-earnings-yield/table/by-year",
"earnings_yield_month": "s-p-500-earnings-yield/table/by-month",
"real_price_year": "s-p-500-historical-prices/table/by-year",
"real_price_month": "s-p-500-historical-prices/table/by-month",
"inflation_adjusted_price_year": "inflation-adjusted-s-p-500/table/by-year",
"inflation_adjusted_price_month": "inflation-adjusted-s-p-500/table/by-month",
"sales_year": "s-p-500-sales/table/by-year",
"sales_quarter": "s-p-500-sales/table/by-quarter",
"sales_growth_year": "s-p-500-sales-growth/table/by-year",
"sales_growth_quarter": "s-p-500-sales-growth/table/by-quarter",
"real_sales_year": "s-p-500-real-sales/table/by-year",
"real_sales_quarter": "s-p-500-real-sales/table/by-quarter",
"real_sales_growth_year": "s-p-500-real-sales-growth/table/by-year",
"real_sales_growth_quarter": "s-p-500-real-sales-growth/table/by-quarter",
"price_to_sales_year": "s-p-500-price-to-sales/table/by-year",
"price_to_sales_quarter": "s-p-500-price-to-sales/table/by-quarter",
"price_to_book_value_year": "s-p-500-price-to-book/table/by-year",
"price_to_book_value_quarter": "s-p-500-price-to-book/table/by-quarter",
"book_value_year": "s-p-500-book-value/table/by-year",
"book_value_quarter": "s-p-500-book-value/table/by-quarter",
}


class MultplSP500MultiplesQueryParams(SP500MultiplesQueryParams):
"""Multpl S&P 500 Multiples Query Params."""

__json_schema_extra__ = {
"series_name": {
"multiple_items_allowed": True,
"choices": sorted(list(URL_DICT)),
}
}

@field_validator("series_name", mode="before", check_fields=False)
@classmethod
def validate_series_name(cls, v):
"""Validate series_name."""
series = v.split(",")
new_values: List = []
for s in series:
if s not in URL_DICT:
raise OpenBBError(
f"{s} is not a valid `series_name`. Choices are: \n{sorted(list(URL_DICT))}\n"
)
new_values.append(s)
if not new_values:
raise OpenBBError(
f"No valid series names provided. Choices are: \n{sorted(list(URL_DICT))}\n"
)
return ",".join(new_values)


class MultplSP500MultiplesData(SP500MultiplesData):
"""Multpl S&P 500 Multiples Data."""

__alias_dict__ = {
"date": "Date",
"value": "Value",
}


class MultplSP500MultiplesFetcher(
Fetcher[
MultplSP500MultiplesQueryParams,
List[MultplSP500MultiplesData],
]
):
"""Multpl S&P 500 Multiples Fetcher."""

@staticmethod
def transform_query(params: Dict[str, Any]) -> MultplSP500MultiplesQueryParams:
"""Transform the query params."""
return MultplSP500MultiplesQueryParams(**params)

@staticmethod
async def aextract_data(
query: MultplSP500MultiplesQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
) -> List[Dict]:
"""Extract data."""
# pylint: disable=import-outside-toplevel
import asyncio # noqa
from io import StringIO
from openbb_core.provider.utils.helpers import amake_request
from numpy import nan
from pandas import read_html, to_datetime

series = query.series_name.split(",")
urls = {s: f"{BASE_URL}{URL_DICT[s]}" for s in series}
results: List = []

async def response_callback(response, _):
"""Response callback."""
return await response.text()

async def get_one(url, series):
"""Get data for one series."""
res = await amake_request(url, response_callback=response_callback)
if res:
df = read_html(StringIO(res))[0] # type: ignore
if not df.empty:
df["Date"] = to_datetime(df["Date"]).dt.date
df = df.sort_values("Date").reset_index(drop=True)
if query.start_date:
df = df[df["Date"] >= query.start_date]
if query.end_date:
df = df[df["Date"] <= query.end_date]
df["Value"] = df["Value"].apply(
lambda x: (
x.strip().replace("† ", "").replace("%", "")
if isinstance(x, str)
else x
)
)
df["name"] = series
if "growth" in series or "yield" in series:
df["Value"] = df["Value"].astype(float) / 100

results.extend(df.replace({nan: None}).to_dict(orient="records"))
else:
warn(f"Failed to get data for {series}.")

await asyncio.gather(*[get_one(url, series) for series, url in urls.items()])

if not results:
raise EmptyDataError("The request was returned empty.")

return results

@staticmethod
def transform_data(
query: MultplSP500MultiplesQueryParams,
data: List[Dict],
**kwargs: Any,
) -> List[MultplSP500MultiplesData]:
"""Transform and validate the data."""
return [
MultplSP500MultiplesData.model_validate(d)
for d in sorted(
data,
key=lambda x: (x["Date"], x["name"]),
)
]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Utilities and Helpers."""
Loading
Loading