Update dependencies and examples (#218)

dask · May 5, 2022 · 5a9bd2b · 5a9bd2b
1 parent b0cbe56
commit 5a9bd2b
Show file tree

Hide file tree

Showing 49 changed files with 1,131 additions and 4,224 deletions.
diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml
@@ -1,41 +1,55 @@
 name: CI
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-deploy:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
 
+      - name: Cache conda
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if binder/environment.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('binder/environment.yml') }}
+
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: "latest"
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
           environment-file: binder/environment.yml
           activate-environment: dask-examples
           auto-activate-base: false
+          use-only-tar-bz2: true
 
       - name: Install testing and docs dependencies
-        shell: bash -l {0}
         run: |
-          conda install -q pathlib nbconvert nbformat jupyter_client ipykernel
-          pip install nbsphinx dask-sphinx-theme>=2 sphinx
+          mamba install nbconvert nbformat jupyter_client ipykernel nbmake \
+          pytest nbsphinx dask-sphinx-theme>=2 sphinx
 
       - name: Execute Notebooks
-        shell: bash -l {0}
         run: |
-          pip install nbmake==0.1 pytest-xdist
           pytest \
-            -n=auto \
+            -vv \
             --nbmake \
             --overwrite \
-            --forked \
             --ignore machine-learning/torch-prediction.ipynb \
             --ignore applications/json-data-on-the-web.ipynb
 
       - name: Build
-        shell: bash -l {0}
         run: sphinx-build -M html . _build -v
 
       - name: Deploy

diff --git a/.github/workflows/ci-repo2docker.yaml b/.github/workflows/ci-repo2docker.yaml
@@ -1,24 +1,25 @@
 name: repo2docker CI
-
 on: [push, pull_request]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Install repo2docker
         run: |
           python -m pip install --upgrade pip
-          python -m pip install jupyter-repo2docker \
-            six chardet # See https://github.com/jupyterhub/repo2docker/issues/1065
-
+          python -m pip install jupyter-repo2docker
       - name: Build dask-examples Docker image
         run: jupyter-repo2docker --no-run --debug .
diff --git a/.github/workflows/ci-update-dependencies.yaml b/.github/workflows/ci-update-dependencies.yaml
@@ -0,0 +1,51 @@
+name: Update Dependencies
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  update-dependencies:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Cache conda
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if binder/environment-base.yml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('binder/environment-base.yml') }}
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          environment-file: binder/environment-base.yml
+          auto-activate-base: false
+          activate-environment: dask-examples
+          use-only-tar-bz2: true
+
+      - name: Install dependencies
+        run: |
+          mamba install nbconvert nbformat jupyter_client ipykernel nbmake pytest-xdist
+
+      - name: Execute Notebooks
+        run: |
+          pytest \
+            -vv \
+            -n=auto \
+            --forked \
+            --nbmake \
+            --overwrite \
+            --ignore machine-learning/torch-prediction.ipynb \
+            --ignore applications/json-data-on-the-web.ipynb
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,11 @@ _build/
 mydask.png
 dataframes/data
 .idea/
+data/
+.vscode/
+applications/*.gif
+applications/*.img
+*.csv
+*.tif
+*.json
+machine-learning/data.zip
diff --git a/README.md b/README.md
@@ -17,30 +17,54 @@ This repository is a great opportunity to start contributing to Dask.
 Please note that examples submitted to this repository should follow these
 guidelines:
 
-1.  Run top-to-bottom without intervention from the user
-2.  Not require external data sources that may disappear over time
-    (external data sources that are highly unlikely to disappear are fine)
-3.  Not be resource intensive, and should run within 2GB of memory
-4.  Be clear and contain enough prose to explain the topic at hand
-5.  Be concise and limited to one or two topics, such that a reader can
-    get through the example within a few minutes of reading
-6.  Be of general relevance to Dask users, and so not too specific on a
-    particular problem or use case
-
-    As an example "how to do dataframe joins" is a great topic while "how to
-    do dataframe joins in the particular case when one column is a categorical
-    and the other is object dtype" is probably too specific
-7.  If the example requires a library not included in `binder/environment.yml`
-    then it would be `pip` installed` in the first cell of the notebook, with a
-    brief explanation about what functionality the library adds.  A brief
-    example follows:
-
-    ```
-    ### Install Extra Dependencies
-
-    We first install the library X for interacting with Y
-    ```
-
-    ```
-    !pip install X
-    ```
+1. Run top-to-bottom without intervention from the user
+2. Not require external data sources that may disappear over time
+   (external data sources that are highly unlikely to disappear are fine)
+3. Not be resource intensive, and should run within 2GB of memory
+4. Be clear and contain enough prose to explain the topic at hand
+5. Be concise and limited to one or two topics, such that a reader can
+   get through the example within a few minutes of reading
+6. Be of general relevance to Dask users, and so not too specific on a
+   particular problem or use case
+
+   As an example "how to do dataframe joins" is a great topic while "how to
+   do dataframe joins in the particular case when one column is a categorical
+   and the other is object dtype" is probably too specific
+7. If the example requires a library not included in `binder/environment.yml`
+   then it would be `pip` installed` in the first cell of the notebook, with a
+   brief explanation about what functionality the library adds.  A brief
+   example follows:
+
+   ```markdown
+   ### Install Extra Dependencies
+
+   We first install the library X for interacting with Y
+   ```
+
+   ```python
+   !pip install X
+   ```
+
+Updating the Binder environment
+-------------------------------
+
+1. Modify `binder/environment-base.yml` with new or updated dependencies
+2. Run a `linux/amd64` Docker container with `mamba` available. For example:
+
+   ```shell
+   docker run --platform=linux/amd64 -it --rm --mount type=bind,source=$(pwd)/binder,target=/binder condaforge/mambaforge /bin/bash
+   ```
+
+   This mounts the `./binder` folder in `/binder` in the Docker container
+3. Create the environment
+
+   ```shell
+   mamba env create -f environment-base.yml
+   ```
+
+   This may take quite a while.
+4. Export the environment specification:
+
+   ```shell
+   mamba env export -n dask-examples --no-builds -f environment.yml
+   ```
diff --git a/applications/async-await.ipynb b/applications/async-await.ipynb
@@ -150,18 +150,11 @@
     "    asyncio.get_event_loop().run_until_complete(f())\n",
     "```"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -175,7 +168,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.9.12"
   }
  },
  "nbformat": 4,

diff --git a/applications/async-web-server.ipynb b/applications/async-web-server.ipynb
@@ -18,7 +18,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -70,7 +69,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -111,7 +109,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -125,7 +122,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -139,7 +135,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -153,7 +148,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -176,7 +170,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -225,7 +218,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -285,7 +277,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -300,7 +291,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -326,7 +316,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -344,7 +333,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -376,7 +364,6 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "collapsed": false,
     "jupyter": {
      "outputs_hidden": false
     }
@@ -395,7 +382,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -409,7 +396,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.4"
+   "version": "3.9.12"
   }
  },
  "nbformat": 4,