diff --git a/examples/quickstart_aws/pyproject.toml b/examples/quickstart_aws/pyproject.toml index fed528d4a7a14..092e7f576cd64 100644 --- a/examples/quickstart_aws/pyproject.toml +++ b/examples/quickstart_aws/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "quickstart_aws" diff --git a/examples/quickstart_aws/quickstart_aws/__init__.py b/examples/quickstart_aws/quickstart_aws/__init__.py index e465d1b3fece2..a22903e7adf79 100644 --- a/examples/quickstart_aws/quickstart_aws/__init__.py +++ b/examples/quickstart_aws/quickstart_aws/__init__.py @@ -1 +1,32 @@ -from .repository import quickstart_aws +from dagster_aws.s3.io_manager import s3_pickle_io_manager +from dagster_aws.s3.resources import s3_resource + +from dagster import ( + Definitions, + ScheduleDefinition, + define_asset_job, + load_assets_from_package_module, +) + +from . import assets + +daily_refresh_schedule = ScheduleDefinition( + job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" +) + + +defs = Definitions( + assets=load_assets_from_package_module(assets), + # The AWS resources use boto under the hood, so if you are accessing your private + # buckets, you will need to provide the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY + # environment variables or follow one of the other boto authentication methods. + # Read about using environment variables and secrets in Dagster: + # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets + resources={ + # With this I/O manager in place, your job runs will store data passed between assets + # on S3 in the location s3:///dagster/storage/. + "io_manager": s3_pickle_io_manager.configured({"s3_bucket": {"env": "S3_BUCKET"}}), + "s3": s3_resource, + }, + schedules=[daily_refresh_schedule], +) diff --git a/examples/quickstart_aws/quickstart_aws/repository.py b/examples/quickstart_aws/quickstart_aws/repository.py deleted file mode 100644 index 85b3683c97173..0000000000000 --- a/examples/quickstart_aws/quickstart_aws/repository.py +++ /dev/null @@ -1,37 +0,0 @@ -from dagster_aws.s3.io_manager import s3_pickle_io_manager -from dagster_aws.s3.resources import s3_resource - -from dagster import ( - ScheduleDefinition, - define_asset_job, - load_assets_from_package_module, - repository, - with_resources, -) - -from . import assets - -daily_refresh_schedule = ScheduleDefinition( - job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" -) - - -@repository -def quickstart_aws(): - return [ - *with_resources( - load_assets_from_package_module(assets), - # The AWS resources use boto under the hood, so if you are accessing your private - # buckets, you will need to provide the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY - # environment variables or follow one of the other boto authentication methods. - # Read about using environment variables and secrets in Dagster: - # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets - resource_defs={ - # With this I/O manager in place, your job runs will store data passed between assets - # on S3 in the location s3:///dagster/storage/. - "io_manager": s3_pickle_io_manager.configured({"s3_bucket": {"env": "S3_BUCKET"}}), - "s3": s3_resource, - }, - ), - daily_refresh_schedule, - ] diff --git a/examples/quickstart_aws/quickstart_aws_tests/test_defs.py b/examples/quickstart_aws/quickstart_aws_tests/test_defs.py new file mode 100644 index 0000000000000..51b8d4cd2f120 --- /dev/null +++ b/examples/quickstart_aws/quickstart_aws_tests/test_defs.py @@ -0,0 +1,5 @@ +from quickstart_aws import defs + + +def test_def_can_load(): + assert defs.get_job_def("all_assets_job") diff --git a/examples/quickstart_aws/workspace.yaml b/examples/quickstart_aws/workspace.yaml deleted file mode 100644 index 6991699b3d595..0000000000000 --- a/examples/quickstart_aws/workspace.yaml +++ /dev/null @@ -1,2 +0,0 @@ -load_from: - - python_package: quickstart_aws diff --git a/examples/quickstart_gcp/pyproject.toml b/examples/quickstart_gcp/pyproject.toml index fed528d4a7a14..4b940ca2d3520 100644 --- a/examples/quickstart_gcp/pyproject.toml +++ b/examples/quickstart_gcp/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "quickstart_gcp" diff --git a/examples/quickstart_gcp/quickstart_gcp/__init__.py b/examples/quickstart_gcp/quickstart_gcp/__init__.py index dbba7b82e7ba0..245a758be5d2f 100644 --- a/examples/quickstart_gcp/quickstart_gcp/__init__.py +++ b/examples/quickstart_gcp/quickstart_gcp/__init__.py @@ -1 +1,29 @@ -from .repository import quickstart_gcp +from dagster import ( + Definitions, + ScheduleDefinition, + define_asset_job, + load_assets_from_package_module, +) + +from . import assets +from .io_managers import bigquery_pandas_io_manager + +daily_refresh_schedule = ScheduleDefinition( + job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" +) + + +defs = Definitions( + assets=load_assets_from_package_module(assets), + resources={ + # Read about using environment variables and secrets in Dagster: + # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets + "io_manager": bigquery_pandas_io_manager.configured( + { + "credentials": {"env": "BIGQUERY_SERVICE_ACCOUNT_CREDENTIALS"}, + "project_id": {"env": "BIGQUERY_PROJECT_ID"}, + } + ), + }, + schedules=[daily_refresh_schedule], +) diff --git a/examples/quickstart_gcp/quickstart_gcp/repository.py b/examples/quickstart_gcp/quickstart_gcp/repository.py deleted file mode 100644 index 7b169c7ded7f5..0000000000000 --- a/examples/quickstart_gcp/quickstart_gcp/repository.py +++ /dev/null @@ -1,34 +0,0 @@ -from dagster import ( - ScheduleDefinition, - define_asset_job, - load_assets_from_package_module, - repository, - with_resources, -) - -from . import assets -from .io_managers import bigquery_pandas_io_manager - -daily_refresh_schedule = ScheduleDefinition( - job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" -) - - -@repository -def quickstart_gcp(): - return [ - *with_resources( - load_assets_from_package_module(assets), - resource_defs={ - # Read about using environment variables and secrets in Dagster: - # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets - "io_manager": bigquery_pandas_io_manager.configured( - { - "credentials": {"env": "BIGQUERY_SERVICE_ACCOUNT_CREDENTIALS"}, - "project_id": {"env": "BIGQUERY_PROJECT_ID"}, - } - ), - }, - ), - daily_refresh_schedule, - ] diff --git a/examples/quickstart_gcp/quickstart_gcp_tests/test_defs.py b/examples/quickstart_gcp/quickstart_gcp_tests/test_defs.py new file mode 100644 index 0000000000000..b0c6d44abd6ae --- /dev/null +++ b/examples/quickstart_gcp/quickstart_gcp_tests/test_defs.py @@ -0,0 +1,5 @@ +from quickstart_gcp import defs + + +def test_def_can_load(): + assert defs.get_job_def("all_assets_job") diff --git a/examples/quickstart_gcp/workspace.yaml b/examples/quickstart_gcp/workspace.yaml deleted file mode 100644 index 03e404e722238..0000000000000 --- a/examples/quickstart_gcp/workspace.yaml +++ /dev/null @@ -1,2 +0,0 @@ -load_from: - - python_package: quickstart_gcp diff --git a/examples/quickstart_snowflake/pyproject.toml b/examples/quickstart_snowflake/pyproject.toml index fed528d4a7a14..b4374f4f7c76d 100644 --- a/examples/quickstart_snowflake/pyproject.toml +++ b/examples/quickstart_snowflake/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "quickstart_snowflake" diff --git a/examples/quickstart_snowflake/quickstart_snowflake/__init__.py b/examples/quickstart_snowflake/quickstart_snowflake/__init__.py index fc1fa6b8b22c1..610f58900051e 100644 --- a/examples/quickstart_snowflake/quickstart_snowflake/__init__.py +++ b/examples/quickstart_snowflake/quickstart_snowflake/__init__.py @@ -1 +1,34 @@ -from .repository import quickstart_snowflake +from dagster_snowflake_pandas import snowflake_pandas_io_manager + +from dagster import ( + Definitions, + ScheduleDefinition, + define_asset_job, + load_assets_from_package_module, +) + +from . import assets + +daily_refresh_schedule = ScheduleDefinition( + job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" +) + + +defs = Definitions( + assets=load_assets_from_package_module(assets), + resources={ + "io_manager": snowflake_pandas_io_manager.configured( + # Read about using environment variables and secrets in Dagster: + # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets + { + "account": {"env": "SNOWFLAKE_ACCOUNT"}, + "user": {"env": "SNOWFLAKE_USER"}, + "password": {"env": "SNOWFLAKE_PASSWORD"}, + "warehouse": {"env": "SNOWFLAKE_WAREHOUSE"}, + "database": {"env": "SNOWFLAKE_DATABASE"}, + "schema": {"env": "SNOWFLAKE_SCHEMA"}, + } + ), + }, + schedules=[daily_refresh_schedule], +) diff --git a/examples/quickstart_snowflake/quickstart_snowflake/repository.py b/examples/quickstart_snowflake/quickstart_snowflake/repository.py deleted file mode 100644 index 5edfcbcdf8168..0000000000000 --- a/examples/quickstart_snowflake/quickstart_snowflake/repository.py +++ /dev/null @@ -1,40 +0,0 @@ -from dagster_snowflake import build_snowflake_io_manager -from dagster_snowflake_pandas import SnowflakePandasTypeHandler - -from dagster import ( - ScheduleDefinition, - define_asset_job, - load_assets_from_package_module, - repository, - with_resources, -) - -from . import assets - -daily_refresh_schedule = ScheduleDefinition( - job=define_asset_job(name="all_assets_job"), cron_schedule="0 0 * * *" -) - - -@repository -def quickstart_snowflake(): - return [ - *with_resources( - load_assets_from_package_module(assets), - resource_defs={ - "io_manager": build_snowflake_io_manager([SnowflakePandasTypeHandler()]).configured( - # Read about using environment variables and secrets in Dagster: - # https://docs.dagster.io/guides/dagster/using-environment-variables-and-secrets - { - "account": {"env": "SNOWFLAKE_ACCOUNT"}, - "user": {"env": "SNOWFLAKE_USER"}, - "password": {"env": "SNOWFLAKE_PASSWORD"}, - "warehouse": {"env": "SNOWFLAKE_WAREHOUSE"}, - "database": {"env": "SNOWFLAKE_DATABASE"}, - "schema": {"env": "SNOWFLAKE_SCHEMA"}, - } - ), - }, - ), - daily_refresh_schedule, - ] diff --git a/examples/quickstart_snowflake/quickstart_snowflake_tests/test_defs.py b/examples/quickstart_snowflake/quickstart_snowflake_tests/test_defs.py new file mode 100644 index 0000000000000..236da155b85b2 --- /dev/null +++ b/examples/quickstart_snowflake/quickstart_snowflake_tests/test_defs.py @@ -0,0 +1,5 @@ +from quickstart_snowflake import defs + + +def test_def_can_load(): + assert defs.get_job_def("all_assets_job") diff --git a/examples/quickstart_snowflake/workspace.yaml b/examples/quickstart_snowflake/workspace.yaml deleted file mode 100644 index d8e5020d704f9..0000000000000 --- a/examples/quickstart_snowflake/workspace.yaml +++ /dev/null @@ -1,2 +0,0 @@ -load_from: - - python_package: quickstart_snowflake