Create a custom configured resource for Strava API

.gitignore

@@ -2,4 +2,6 @@
 logs
 .env
 dagster_proj.egg-info
-.vscode
+.vscode
+*.pyc
+__pyache__

dagster_proj/__init__.py

@@ -1,7 +1,7 @@
 from dagster import Definitions, load_assets_from_modules
 
 from .assets import activities, dbt, energy_prediction
-from .resources import database_resource, dbt_resource
+from .resources import database_resource, dbt_resource, strava_api_resouce
 from .jobs import activities_update_job
 from .schedules import activities_update_schedule
 
@@ -17,6 +17,7 @@
     resources={
         "database": database_resource,
         "dbt": dbt_resource,
+        'strava': strava_api_resouce,
     },
     jobs=all_jobs,
     schedules=all_schedules,

dagster_proj/assets/activities.py

@@ -12,30 +12,14 @@
 import pendulum
 import sys
 
-logger = get_dagster_logger()
-
-CLIENT_ID = EnvVar("CLIENT_ID").get_value()
-CLIENT_SECRET = EnvVar("CLIENT_SECRET").get_value()
-REFRESH_TOEKEN = EnvVar("REFRESH_TOKEN").get_value()
-
+from ..resources import StravaAPIResource, strava_api_resouce
 
-def strava_access_token(refresh_token, client_id, client_secret):
-    """Return the access_token for Authorization bearer"""
-    auth_url = "https://www.strava.com/oauth/token"
-    payload = {
-        'client_id': client_id,
-        'client_secret': client_secret,
-        'refresh_token': refresh_token,
-        'grant_type': 'refresh_token'
-    }
-    response = requests.post(auth_url, data=payload)
-    access_token = response.json()['access_token']
-    return access_token
+logger = get_dagster_logger()
 
 @dlt.source
-def strava_rest_api_config(client_id, client_secret, refresh_token):
+def strava_rest_api_config(strava_resource: StravaAPIResource):
     logger.info("Extracting Strava data source")
-    access_token = strava_access_token(refresh_token, client_id, client_secret)
+    access_token = strava_api_resouce.get_access_token()
 
     config: RESTAPIConfig = {
         "client": {
@@ -85,11 +69,7 @@ def load_strava_activities():
         dataset_name="activities",
         progress="log")
 
-    source = strava_rest_api_config(
-        client_id=CLIENT_ID,
-        client_secret=CLIENT_SECRET,
-        refresh_token=REFRESH_TOEKEN
-    )
+    source = strava_rest_api_config(strava_api_resouce)
 
     load_info = pipeline.run(source)
     logger.info(load_info)

dagster_proj/resources/__init__.py

@@ -3,11 +3,19 @@
 from dagster_dbt import DbtCliResource
 from ..project import dbt_project
 
+from .configured_resources import StravaAPIResource
+
 dbt_resource = DbtCliResource(
     project_dir=dbt_project,
     profiles_dir='analytics_dbt',
 )
 
 database_resource = DuckDBResource(
     database=EnvVar("DUCKDB_DATABASE"),
-)
+)
+
+strava_api_resouce = StravaAPIResource(
+    client_id=EnvVar("CLIENT_ID").get_value(),
+    client_secret=EnvVar("CLIENT_SECRET").get_value(),
+    refresh_token=EnvVar("REFRESH_TOKEN").get_value(),
+)

dagster_proj/resources/configured_resources.py

@@ -0,0 +1,27 @@
+from dagster import ConfigurableResource, asset, Definitions
+import requests
+from typing import Dict, Any
+
+class StravaAPIResource(ConfigurableResource):
+    client_id: str
+    client_secret: str
+    refresh_token: str
+    _access_token: str = None
+
+    def get_access_token(self) -> str:
+        if not self._access_token:
+            self._access_token = self._refresh_access_token()
+        return self._access_token
+
+    def _refresh_access_token(self) -> str:
+        auth_url = "https://www.strava.com/oauth/token"
+        payload = {
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'refresh_token': self.refresh_token,
+            'grant_type': 'refresh_token'
+        }
+        response = requests.post(auth_url, data=payload, timeout=100)
+        response.raise_for_status() 
+        return response.json()['access_token']
+

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "dagster-dbt>=0.25.3",
     "numpy>=2.0.2",
     "scikit-learn>=1.6.0",
+    "dagster-cloud>=1.9.3",
 ]
 
 [project.optional-dependencies]
@@ -30,7 +31,5 @@ build-backend = "setuptools.build_meta"
 module_name = "dagster_proj"
 code_location_name = "dagster_proj"
 
-[dependency-groups]
-dev = [
-    "ipykernel>=6.29.5",
-]
+[tool.setuptools]
+packages = ["dagster_proj"]

requirements.txt

@@ -32,6 +32,8 @@ cachetools==5.5.0
     # via streamlit
 certifi==2024.12.14
     # via requests
+cffi==1.17.1
+    # via cryptography
 charset-normalizer==3.4.0
     # via requests
 click==8.1.7
@@ -50,15 +52,23 @@ coloredlogs==14.0
     # via dagster
 croniter==3.0.4
     # via dagster
+cryptography==44.0.0
+    # via pyjwt
 daff==1.3.46
     # via dbt-core
 dagster==1.9.5
     # via
     #   dagster-proj (pyproject.toml)
+    #   dagster-cloud
+    #   dagster-cloud-cli
     #   dagster-dbt
     #   dagster-duckdb
     #   dagster-graphql
     #   dagster-webserver
+dagster-cloud==1.9.5
+    # via dagster-proj (pyproject.toml)
+dagster-cloud-cli==1.9.5
+    # via dagster-cloud
 dagster-dbt==0.25.5
     # via dagster-proj (pyproject.toml)
 dagster-duckdb==0.25.5
@@ -90,6 +100,12 @@ dbt-semantic-interfaces==0.5.1
     # via dbt-core
 deepdiff==7.0.1
     # via dbt-common
+deprecated==1.2.15
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-semantic-conventions
 dlt==1.4.1
     # via dagster-proj (pyproject.toml)
 docstring-parser==0.16
@@ -107,12 +123,18 @@ fsspec==2024.10.0
     #   universal-pathlib
 gitdb==4.0.11
     # via gitpython
+github3-py==4.0.1
+    # via dagster-cloud-cli
 gitpython==3.1.43
     # via
     #   dlt
     #   streamlit
 giturlparse==0.12.0
     # via dlt
+googleapis-common-protos==1.66.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 gql==3.5.0
     # via dagster-graphql
 graphene==3.4.3
@@ -128,6 +150,7 @@ grpcio==1.68.1
     # via
     #   dagster
     #   grpcio-health-checking
+    #   opentelemetry-exporter-otlp-proto-grpc
 grpcio-health-checking==1.68.1
     # via dagster
 h11==0.14.0
@@ -146,7 +169,9 @@ idna==3.10
     #   requests
     #   yarl
 importlib-metadata==6.11.0
-    # via dbt-semantic-interfaces
+    # via
+    #   dbt-semantic-interfaces
+    #   opentelemetry-api
 isodate==0.6.1
     # via
     #   agate
@@ -214,6 +239,33 @@ numpy==2.2.0
     #   scikit-learn
     #   scipy
     #   streamlit
+opentelemetry-api==1.29.0
+    # via
+    #   dagster-cloud
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp-proto-common==1.29.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.29.0
+    # via dagster-cloud
+opentelemetry-exporter-otlp-proto-http==1.29.0
+    # via dagster-cloud
+opentelemetry-proto==1.29.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.29.0
+    # via
+    #   dagster-cloud
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.50b0
+    # via opentelemetry-sdk
 ordered-set==4.1.0
     # via deepdiff
 orjson==3.10.12
@@ -224,6 +276,7 @@ packaging==24.2
     # via
     #   altair
     #   dagster
+    #   dagster-cloud-cli
     #   dagster-dbt
     #   dbt-core
     #   dlt
@@ -241,12 +294,16 @@ pathvalidate==3.2.1
     # via dlt
 pendulum==3.0.0
     # via dlt
+pex==2.27.1
+    # via dagster-cloud
 pillow==11.0.0
     # via streamlit
 pluggy==1.5.0
     # via dlt
 ply==3.11
     # via jsonpath-ng
+prompt-toolkit==3.0.36
+    # via questionary
 propcache==0.2.1
     # via yarl
 protobuf==5.29.1
@@ -255,10 +312,14 @@ protobuf==5.29.1
     #   dbt-adapters
     #   dbt-common
     #   dbt-core
+    #   googleapis-common-protos
     #   grpcio-health-checking
+    #   opentelemetry-proto
     #   streamlit
 pyarrow==18.1.0
     # via streamlit
+pycparser==2.22
+    # via cffi
 pydantic==2.10.3
     # via
     #   dagster
@@ -269,11 +330,14 @@ pydeck==0.9.1
     # via streamlit
 pygments==2.18.0
     # via rich
+pyjwt==2.10.1
+    # via github3-py
 python-dateutil==2.9.0.post0
     # via
     #   croniter
     #   dbt-common
     #   dbt-semantic-interfaces
+    #   github3-py
     #   graphene
     #   pandas
     #   pendulum
@@ -297,24 +361,33 @@ pytz==2024.2
 pyyaml==6.0.2
     # via
     #   dagster
+    #   dagster-cloud-cli
     #   dbt-core
     #   dbt-semantic-interfaces
     #   dlt
     #   uvicorn
+questionary==2.0.1
+    # via
+    #   dagster-cloud
+    #   dagster-cloud-cli
 referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
 requests==2.32.3
     # via
     #   dagster
+    #   dagster-cloud
+    #   dagster-cloud-cli
     #   dagster-dbt
     #   dagster-graphql
     #   dbt-common
     #   dbt-core
     #   dlt
+    #   github3-py
     #   gql
     #   minimal-snowplow-tracker
+    #   opentelemetry-exporter-otlp-proto-http
     #   requests-toolbelt
     #   streamlit
 requests-toolbelt==1.0.0
@@ -397,7 +470,10 @@ tornado==6.4.2
 tqdm==4.67.1
     # via dagster
 typer==0.15.1
-    # via dagster-dbt
+    # via
+    #   dagster-cloud
+    #   dagster-cloud-cli
+    #   dagster-dbt
 types-setuptools==75.6.0.20241126
     # via requirements-parser
 typing-extensions==4.12.2
@@ -413,6 +489,7 @@ typing-extensions==4.12.2
     #   dlt
     #   graphene
     #   mashumaro
+    #   opentelemetry-sdk
     #   pydantic
     #   pydantic-core
     #   sqlalchemy
@@ -425,6 +502,8 @@ tzdata==2024.2
     #   pendulum
 universal-pathlib==0.2.6
     # via dagster
+uritemplate==4.1.1
+    # via github3-py
 urllib3==2.2.3
     # via requests
 uvicorn==0.34.0
@@ -435,8 +514,12 @@ watchdog==5.0.3
     # via dagster
 watchfiles==1.0.3
     # via uvicorn
+wcwidth==0.2.13
+    # via prompt-toolkit
 websockets==14.1
     # via uvicorn
+wrapt==1.17.0
+    # via deprecated
 yarl==1.18.3
     # via gql
 zipp==3.21.0