Merge pull request #1 from dagster-io/master

Update forked repository.

.buildkite/pipeline.py

@@ -141,7 +141,8 @@ def python_modules_tox_tests(directory, supported_pythons=None):
                 "buildkite-agent artifact upload {file}".format(file=coverage),
             )
             .on_integration_image(
-                version, ['AWS_DEFAULT_REGION', 'TWILIO_TEST_ACCOUNT_SID', 'TWILIO_TEST_AUTH_TOKEN']
+                version,
+                ['AWS_DEFAULT_REGION', 'TWILIO_TEST_ACCOUNT_SID', 'TWILIO_TEST_AUTH_TOKEN',],
             )
             .build()
         )
@@ -321,7 +322,7 @@ def flyte_tests():
             StepBuilder("dagster-flyte tests ({ver})".format(ver=TOX_MAP[version]))
             .run(
                 "pushd python_modules/libraries/dagster-flyte",
-                "tox -e {ver}".format(ver=TOX_MAP[version]),
+                "tox -vv -e {ver}".format(ver=TOX_MAP[version]),
                 "mv .coverage {file}".format(file=coverage),
                 "buildkite-agent artifact upload {file}".format(file=coverage),
             )

.gitpod.Dockerfile

@@ -0,0 +1,9 @@
+FROM gitpod/workspace-full
+
+USER gitpod
+
+# Install wxPython dependencies
+RUN sudo apt-get -q update
+
+ENV DAGSTER_HOME="$HOME/dagster_home"
+RUN mkdir -p $DAGSTER_HOME

.gitpod.yml

@@ -0,0 +1,16 @@
+image:
+  file: .gitpod.Dockerfile
+
+# This will expose all necessary ports needed for your VNC image
+ports:
+  - port: 3000
+    onOpen: open-preview
+
+tasks:
+  - command: |
+      cd examples/$EXAMPLE
+      pip3 install -r requirements.txt
+      [ -e README.md ] && gp open README.md
+      clear && dagit
+  - openMode: split-right
+    command: echo SplitTerminal && clear

.pylintrc

@@ -14,8 +14,9 @@
 # C - convention related checks
 # W0511 disable TODO warning
 # W1201, W1202 disable log format warning. False positives (I think)
+# W0231 disable super-init-not-called - pylint doesn't understand six.with_metaclass(ABCMeta)
 
-disable=C,R,duplicate-code,W0511,W1201,W1202,no-init
+disable=C,R,duplicate-code,W0231,W0511,W1201,W1202,no-init
 
 # See: https://github.com/getsentry/responses/issues/74
 [TYPECHECK]

CHANGES.md

@@ -8,6 +8,8 @@
 - The CLI option `--celery-base-priority` is no longer available for the command:
   `dagster pipeline backfill`. Use the tags option to specify the celery priority, (e.g.
   `dagster pipeline backfill my_pipeline --tags '{ "dagster-celery/run_priority": 3 }'`
+- The `ScheduleExecutionContext` no longer has a repository definition available as a property on the context. If you were previously using it in a schedule definition, inline the repository name instead of retrieving it from the schedule execution context.
+- `@scheduler` and `@repository_partitions` have been removed. Load `ScheduleDefinition`s and `PartitionSetDefinition`s via `RepositoryDefinition` instead.
 
 ## 0.7.13
 

Makefile

@@ -72,7 +72,7 @@ install_dev_python_modules:
 	-pip install -e examples[full] $(QUIET)
 
 	# NOTE: This installation will fail for Python 2.7 (Dask doesn't work w/ py27 on macOS)
-	-pip install -e python_modules/libraries/dagster-dask $(QUIET)
+	-pip install -e "python_modules/libraries/dagster-dask[yarn,pbs,kube]" $(QUIET)
 
 	# This fails on py2
 	-pip install -r .read-the-docs-requirements.txt

bin/assets/dagster-example-tmpl/README.md.tmpl

@@ -0,0 +1,23 @@
+---
+title: {{EXAMPLE_NAME}}
+description: Fill in a description
+---
+
+# {{EXAMPLE_NAME}}
+
+Description of example goes here
+
+# Open in Playground
+
+Open up this example in a playground using [Gitpod](https://gitpod.io)
+
+[![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#EXAMPLE={{EXAMPLE_NAME}}/https://github.com/dagster-io/dagster)
+
+# Download Manually
+
+Download the example:
+
+```
+curl https://codeload.github.com/dagster-io/dagster/tar.gz/master | tar -xz --strip=2 dagster-master/examples/{{EXAMPLE_NAME}}
+cd dep_dsl
+```

bin/assets/dagster-example-tmpl/repo.py.tmpl

@@ -0,0 +1,15 @@
+from dagster import RepositoryDefinition, pipeline, solid
+
+
+@solid
+def hello(_):
+    return 1
+
+
+@pipeline
+def my_pipeline():
+    hello()
+
+
+def define_repository():
+    return RepositoryDefinition("{{EXAMPLE_NAME}}", pipeline_defs=[my_pipeline])

bin/assets/dagster-example-tmpl/repository.yaml

@@ -0,0 +1,3 @@
+repository:
+  file: repo.py
+  fn: define_repository

bin/assets/dagster-example-tmpl/requirements.txt

@@ -0,0 +1,2 @@
+dagster
+dagit

bin/create_example.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+"""
+To use, run
+ python bin/create_example.py --name example_name
+"""
+from __future__ import absolute_import
+
+import os
+import shutil
+import sys
+
+import click
+
+BASE_PATH = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, BASE_PATH)
+
+from git_utils import get_most_recent_git_tag  # isort:skip
+
+
+def copy_directory(src, dest):
+    try:
+        shutil.copytree(src, dest, ignore=shutil.ignore_patterns('.DS_Store'))
+    # Directories are the same
+    except shutil.Error as e:
+        print('Directory not copied. Error: %s' % e)
+    # Any error saying that the directory doesn't exist
+    except OSError as e:
+        print('Directory not copied. Error: %s' % e)
+
+
+@click.command()
+@click.option('--name', prompt='Name of library', help='Name of library')
+def main(name):
+    template_library_path = os.path.abspath('bin/assets/dagster-example-tmpl')
+    new_template_library_path = os.path.abspath('examples/{name}'.format(name=name))
+
+    if os.path.exists(new_template_library_path):
+        raise click.UsageError('Example with name {name} already exists'.format(name=name))
+
+    copy_directory(template_library_path, new_template_library_path)
+
+    version = get_most_recent_git_tag()
+
+    for dname, _, files in os.walk(new_template_library_path):
+        for fname in files:
+            fpath = os.path.join(dname, fname)
+            with open(fpath) as f:
+                s = f.read()
+            s = s.replace('{{EXAMPLE_NAME}}', name)
+            s = s.replace('{{VERSION}}', version)
+            with open(fpath, 'w') as f:
+                f.write(s)
+
+            new_fname = fname.replace('.tmpl', '')
+            new_fpath = os.path.join(dname, new_fname)
+            shutil.move(fpath, new_fpath)
+            print('Created {path}'.format(path=new_fpath))
+
+        new_dname = dname.replace('example-tmpl', name)
+        shutil.move(dname, new_dname)
+
+    print('Example created at {path}'.format(path=new_template_library_path))
+
+
+if __name__ == "__main__":
+    main()  # pylint: disable=no-value-for-parameter

bin/dagster_module_publisher.py

@@ -93,7 +93,7 @@ def check_for_cruft(self, autoclean):
 
         found_pyc_files = []
 
-        for root, dir_, files in os.walk(git_repo_root()):
+        for root, _, files in os.walk(git_repo_root()):
             for file_ in files:
                 if file_.endswith('.pyc'):
                     found_pyc_files.append(os.path.join(root, file_))
@@ -136,7 +136,6 @@ def check_directory_structure(self):
         expected_modules_not_found = []
 
         module_directories = get_core_module_directories()
-        library_directories = get_library_module_directories()
 
         for module_dir in module_directories:
             if module_dir.name not in expected_python_modules_subdirectories:

docs/Makefile

@@ -151,8 +151,8 @@ buildnext:
 	NODE_ENV=$(or $(NODE_ENV), development) yarn --cwd next sitemap
 
 root_build:
-	NODE_ENV=production make build_next_default version=0.7.12
-	NODE_ENV=production make build_next_version version=0.7.12
+	NODE_ENV=production make build_next_default version=0.7.14
+	NODE_ENV=production make build_next_version version=0.7.14
 
 build_next_master:
 	yarn --cwd next --production=false

docs/README.md

@@ -26,23 +26,11 @@ linkchecker http://127.0.0.1:8000
 NODE_ENV=production make buildnext
 ```
 
-2. Update version links in `next/src/pages/versions/index.mdx` such as (replace `<new_version>` with e.g. `0.7.12`):
+2. Update version links in `next/src/pages/versions/index.mdx`, run:
 
 ```
-# Dagster Versions
-
-## Current version (Stable)
-
-| **<new_version>** | [Documentation](/docs/install) | [Release Notes](https://github.com/dagster-io/dagster/releases/tag/<new_version>) |
-| ----------- | ------------------------------------- | --------------------------------------------------------------------------- | --------------------------------------- |
-
-
-## All Versions
-
-| **<new_version>** | [Documentation](/<new_version>/docs/install)                     | [Release Notes](https://github.com/dagster-io/dagster/releases/tag/<new_version>) |                                         |
-| ----------- | ------------------------------------- | --------------------------------------------------------------------------- | --------------------------------------- |
-| **0.7.11**  | [Documentation](/0.7.11/docs/install) | [Release Notes](https://github.com/dagster-io/dagster/releases/tag/0.7.11)  |                                         |
-| **0.7.10**  | [Documentation](/0.7.10/docs/install) | [Release Notes](https://github.com/dagster-io/dagster/releases/tag/0.7.10)  |                                         |
+cd next
+yarn update-version <new_version> # e.g. `yarn update-version 0.7.12`
 ```
 
 3. Build the static doc site by running `make root_build` in the `docs` directory. It will generate a folder called `out` in the `docs` directory.