docs(tutorial): deploy a jupyterlite repl on the website (#9009)

.github/workflows/ibis-docs-main.yml

@@ -45,6 +45,9 @@ jobs:
       - name: build docs
         run: nix develop --ignore-environment --keep HOME -c just docs-render
 
+      - name: build jupyterlite
+        run: nix develop --ignore-environment --keep HOME -c just build-jupyterlite
+
       - name: check that all frozen computations were done before push
         run: git diff --exit-code --stat
 

.github/workflows/ibis-docs-pr.yml

@@ -47,6 +47,9 @@ jobs:
       - name: build docs
         run: nix develop --ignore-environment --keep HOME -c just docs-render
 
+      - name: build jupyterlite
+        run: nix develop --ignore-environment --keep HOME -c just build-jupyterlite
+
       - name: check that all frozen computations were done before push
         run: git diff --exit-code --stat
 

.gitignore

@@ -147,3 +147,7 @@ docs/**/.jupyter_cache
 # quarto generated files
 docs/posts-listing.json
 docs/posts.feed-full-staged
+docs/**/*.html
+
+# jupyterlite stuff
+.jupyterlite.doit.db

docs/_quarto.yml

@@ -107,6 +107,7 @@ website:
       contents:
         - install.qmd
         - auto: tutorials/*.qmd
+        - auto: tutorials/browser
         - auto: tutorials/cloud-data-platforms
         - auto: tutorials/open-source-software
     - id: concepts

docs/_tabsets/repl_warning.qmd

@@ -0,0 +1,9 @@
+::: {.callout-warning collapse="true"}
+## Ibis in the browser is experimental.
+
+iOS is known to cause crashes on this page.
+
+Mobile Firefox may also not work (the page won't crash though).
+
+Please [open an issue on GitHub](https://github.com/ibis-project/ibis/issues/new/choose) if you encounter problems.
+:::

docs/ibis-wasm-wheels_cors_config.json

@@ -0,0 +1,8 @@
+[
+  {
+    "origin": ["*"],
+    "method": ["GET"],
+    "responseHeader": ["Content-Type"],
+    "maxAgeSeconds": 3600
+  }
+]

docs/jupyter_lite_config.json

@@ -0,0 +1,8 @@
+{
+  "PipliteAddon": {
+    "piplite_urls": [
+      "https://duckdb.github.io/duckdb-pyodide/wheels/duckdb-0.10.2-cp311-cp311-emscripten_3_1_46_wasm32.whl",
+      "dist/ibis_framework-8.0.0-py3-none-any.whl"
+    ]
+  }
+}

docs/styles.css

@@ -14,3 +14,10 @@ section[id^="parameters-"] {
   vertical-align: -0.125em;
   width: 1em;
 }
+
+#jupyterlite-console {
+  width: 100%;
+  height: 95%;
+  margin: auto;
+  display: block;
+}

docs/tutorials/browser/repl.qmd

@@ -0,0 +1,44 @@
+---
+title: JupyterLite console
+format:
+  html:
+    page-layout: full
+---
+
+Try our experimental JupyterLite console with Ibis, using the Palmer
+penguins[^1] dataset loaded into the DuckDB backend!
+
+{{< include ../../_tabsets/repl_warning.qmd >}}
+
+```{python}
+#| echo: false
+#| output: asis
+from urllib.parse import urlencode
+
+lines = """
+%pip install numpy pandas tzdata
+import pyodide_js, pathlib, js
+await pyodide_js.loadPackage("https://storage.googleapis.com/ibis-wasm-wheels/pyarrow-16.0.0.dev2661%2Bg9bddb87fd-cp311-cp311-emscripten_3_1_46_wasm32.whl")
+pathlib.Path("penguins.csv").write_text(await (await js.fetch("https://storage.googleapis.com/ibis-tutorial-data/penguins.csv")).text())
+del pyodide_js, pathlib, js
+%clear
+%pip install 'ibis-framework[duckdb]'
+from ibis.interactive import *
+penguins = ibis.read_csv("penguins.csv")
+penguins
+"""
+
+params = [
+    ("toolbar", "1"),
+    ("theme", "JupyterLab Night"),
+    ("kernel", "python"),
+]
+params.extend(("code", line) for line in lines.splitlines() if line)
+query = urlencode(params)
+
+jupyterlite = f"../../jupyterlite/repl/?{query}"
+iframe = f'<iframe src="{jupyterlite}" id="jupyterlite-console"></iframe>'
+print(iframe)
+```
+
+[^1]: Horst AM, Hill AP, Gorman KB (2020). palmerpenguins: Palmer Archipelago (Antarctica) penguin data. R package version 0.1.0. <https://allisonhorst.github.io/palmerpenguins/>. doi: 10.5281/zenodo.3960218.

gen_redirects.py

@@ -169,6 +169,7 @@
     "/versioning": "/concepts/versioning",
     "/why_ibis/": "/why",
     "/tutorials/data-platforms/": "tutorials/cloud-data-platforms/",
+    "/repl": "/tutorials/browser/repl",
     # TODO: "/how_to/memtable-join/": "/how_to/memtable_join/",
     # TODO: "/how_to/memtable_join/"
     # TODO: "/reference/backends/base/"

justfile

@@ -9,6 +9,8 @@ clean:
 # lock dependencies without updating existing versions
 lock:
     #!/usr/bin/env bash
+    set -euo pipefail
+
     required_version="1.8.2"
     version="$(poetry --version)"
     if ! grep -qF "${required_version}" <<< "${version}"; then
@@ -39,7 +41,7 @@ ci-check *args:
 # lint code
 lint:
     ruff format -q . --check
-    ruff .
+    ruff check .
 
 # run the test suite for one or more backends
 test +backends:
@@ -56,6 +58,7 @@ test +backends:
 
 _doctest runner *args:
     #!/usr/bin/env bash
+    set -euo pipefail
 
     # TODO(cpcloud): why doesn't pytest --ignore-glob=test_*.py work?
     {{ runner }} pytest --doctest-modules {{ args }} $(
@@ -80,6 +83,8 @@ ci-doctest *args:
 # download testing data
 download-data owner="ibis-project" repo="testing-data" rev="master":
     #!/usr/bin/env bash
+    set -euo pipefail
+
     outdir="{{ justfile_directory() }}/ci/ibis-testing-data"
     rm -rf "$outdir"
     url="https://github.com/{{ owner }}/{{ repo }}"
@@ -109,6 +114,8 @@ reup *backends:
 # stop and remove containers; clean up networks and volumes
 down *backends:
     #!/usr/bin/env bash
+    set -euo pipefail
+
     if [ -z "{{ backends }}" ]; then
         docker compose down --volumes --remove-orphans
     else
@@ -130,11 +137,15 @@ benchcmp number *args:
 # check for invalid links in a locally built version of the docs
 checklinks *args:
     #!/usr/bin/env bash
+    set -euo pipefail
+
     lychee --base docs/_output $(find docs/_output -name '*.html') {{ args }}
 
 # view the changelog for upcoming release (use --pretty to format with glow)
 view-changelog flags="":
     #!/usr/bin/env bash
+    set -euo pipefail
+
     npx -y -p conventional-changelog-cli \
         -- conventional-changelog --config ./.conventionalcommits.js \
         | ([ "{{ flags }}" = "--pretty" ] && glow -p - || cat -)
@@ -169,10 +180,40 @@ docs-api-preview:
 docs-deploy:
     quarto publish --no-prompt --no-browser --no-render netlify docs
 
+# build an ibis_framework wheel that works with pyodide
+build-ibis-for-pyodide:
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    # TODO(cpcloud): remove when:
+    # 1. pyarrow release contains pyodide
+    # 2. ibis supports this version of pyarrow
+    rm -rf dist/
+    poetry add 'pyarrow>=10.0.1' --allow-prereleases
+    poetry build --format wheel
+    git checkout poetry.lock pyproject.toml
+
+# build the jupyterlite deployment
+build-jupyterlite: build-ibis-for-pyodide
+    #!/usr/bin/env bash
+    set -euo pipefail
+
+    mkdir -p docs/_output/jupyterlite
+    jupyter lite build \
+        --debug \
+        --no-libarchive \
+        --config docs/jupyter_lite_config.json \
+        --output-dir docs/_output/jupyterlite
+    # jupyter lite build can copy from the nix store, and preserves the
+    # original write bit; without this the next run of this rule will result in
+    # a permission error when the build tries to remove existing files
+    chmod -R u+w docs/_output/jupyterlite
+
 # run the entire docs build pipeline
 docs-build-all:
     just docs-apigen --verbose
     just docs-render
+    just build-jupyterlite
     just checklinks docs/_output --offline --no-progress
 
 # open chat