From 5a8ce763b80f8b84ec96d0c9774bd2b82cf529eb Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 21:30:05 -0500 Subject: [PATCH 01/12] doit config --- .gitignore | 2 +- dodo.py | 28 ++++++++++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index bb6c1f7d..ab0c674d 100644 --- a/.gitignore +++ b/.gitignore @@ -104,4 +104,4 @@ venv.bak/ .mypy_cache/ #doit -.doit.db* \ No newline at end of file +.doit* diff --git a/dodo.py b/dodo.py index e93215b1..4f0a4c69 100644 --- a/dodo.py +++ b/dodo.py @@ -4,6 +4,14 @@ """ # change the urls above to link different jupyter references. + +DOIT_CONFIG = { + "backend": "sqlite3", + "verbosity": 2, + "par_type": "thread", +} + + import pathlib import doit repos = list(filter(bool, __doc__.splitlines())) @@ -33,7 +41,7 @@ def task_build(): file_dep=[dep], actions=[ f"""cd {dep.parent} && jlpm --prefer-offline --ignore-optional""" - ], + ], targets=[dep.parent / "node_modules/.yarn-integrity"] ) yield dict( @@ -41,7 +49,7 @@ def task_build(): file_dep=[dep.parent / "node_modules/.yarn-integrity"], actions=[ f"""cd {dep.parent} && jlpm build""" - ], + ], targets=list(dep.parent.rglob("packages/*/lib/*.js")) ) @@ -53,23 +61,23 @@ def task_link_lumino(): yield dict( name=f"link_lumino_{pkg.parent.name}", file_dep=[ - pkg, + pkg, pkg.parent.parent.parent / "node_modules/.yarn-integrity" - ], + ], actions=f""" cd lumino/packages/{pkg.parent.name} \ - && jlpm link - + && jlpm link + """.strip().splitlines(),#&& touch ../../build/link.lumino.{pkg.parent.name}.ok targets=[]# [f"build/link.lumino.{pkg.parent.name}.ok"] ) yield dict( name=f"link_lab_{pkg.parent.name}", - file_dep=[],#[f"build/link.lumino.{pkg.parent.name}.ok"], + file_dep=[],#[f"build/link.lumino.{pkg.parent.name}.ok"], actions=f""" cd jupyterlab \ - && jlpm link @lumino/{pkg.parent.name} - + && jlpm link @lumino/{pkg.parent.name} + """.strip().splitlines(), #&& touch ../build/link.lab.{pkg.parent.name}.ok targets=[f"build/link.lab.{pkg.parent.name}.ok"] ) @@ -91,4 +99,4 @@ def task_config(): # actions=[ # doit.CmdAction([], shell=False, cwd=postBuild.parent) # ] -# ) \ No newline at end of file +# ) From 261f78b3f02b5d893792f82b8cdf0e7252cd335f Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 21:32:16 -0500 Subject: [PATCH 02/12] more env stuff --- .gitignore | 2 +- environment.yml | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index ab0c674d..9673f5b8 100644 --- a/.gitignore +++ b/.gitignore @@ -103,5 +103,5 @@ venv.bak/ # mypy .mypy_cache/ -#doit +# doit .doit* diff --git a/environment.yml b/environment.yml index eb31956c..02997687 100644 --- a/environment.yml +++ b/environment.yml @@ -2,18 +2,18 @@ # across different development versions. name: a11y channels: -- conda-forge -- nodefaults + - conda-forge + - nodefaults dependencies: -- nodejs>=12,<15,!=13.* -- python>=3.7,<3.10 -- jupyter-server-proxy -- matplotlib-base -- numpy -- pip -- vega_datasets -- xeus-python -- yarn<2 -- doit -- pip -- pip: {} \ No newline at end of file + - black + - doit + - git + - jupyter-server-proxy + - matplotlib-base + - nodejs>=12,<15,!=13.* + - numpy + - pip + - python>=3.7,<3.10 + - vega_datasets + - xeus-python + - yarn<2 From aa78d616eb303edfd1289153d2d354e3d51bcedb Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 21:36:22 -0500 Subject: [PATCH 03/12] blacken --- dodo.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/dodo.py b/dodo.py index 4f0a4c69..afcd4a35 100644 --- a/dodo.py +++ b/dodo.py @@ -5,20 +5,28 @@ # change the urls above to link different jupyter references. +import pathlib +import doit + DOIT_CONFIG = { "backend": "sqlite3", "verbosity": 2, "par_type": "thread", } +HERE = pathlib.Path(__file__).parent -import pathlib -import doit repos = list(filter(bool, __doc__.splitlines())) -def repo_to_path(x): - """extract the local checkout name""" - return pathlib.Path(x.rpartition("@")[0].rpartition("/")[2]) + +def task_lint(): + all_py = HERE.glob("*.py") + yield dict( + name="py", + file_dep=[*all_py], + actions=[["black", "--silent", *all_py]] + ) + # add targets to the docstring to include in the dev build. def task_clone(): @@ -35,7 +43,6 @@ def task_clone(): def task_build(): """ensure a working build of live development builds""" for dep in [repo_to_path(x) / "package.json" for x in repos]: - print(dep) yield dict( name=f"install_{dep}", file_dep=[dep], @@ -100,3 +107,9 @@ def task_config(): # doit.CmdAction([], shell=False, cwd=postBuild.parent) # ] # ) + +# utilities + +def repo_to_path(x): + """extract the local checkout name""" + return pathlib.Path(x.rpartition("@")[0].rpartition("/")[2]) From 88f8716e07d4527f010ca4517a58b2da8f6fc95b Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 21:38:14 -0500 Subject: [PATCH 04/12] blacken --- dodo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dodo.py b/dodo.py index afcd4a35..85227cd6 100644 --- a/dodo.py +++ b/dodo.py @@ -24,7 +24,7 @@ def task_lint(): yield dict( name="py", file_dep=[*all_py], - actions=[["black", "--silent", *all_py]] + actions=[["black", "--quiet", *all_py]] ) From 66f1e80123af08d0bd41756f2f3f2d0107d9c280 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 23:11:17 -0500 Subject: [PATCH 05/12] clean up linking --- .gitignore | 4 ++ dodo.py | 128 ++++++++++++++++++++++++++++++++---------------- environment.yml | 6 +++ 3 files changed, 96 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 9673f5b8..5dc15646 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,7 @@ venv.bak/ # doit .doit* + +# yarn +.yarn-links/ +node_modules/ diff --git a/dodo.py b/dodo.py index 85227cd6..a667b696 100644 --- a/dodo.py +++ b/dodo.py @@ -5,8 +5,10 @@ # change the urls above to link different jupyter references. +import os import pathlib -import doit +import doit.tools +import json DOIT_CONFIG = { "backend": "sqlite3", @@ -14,9 +16,23 @@ "par_type": "thread", } +os.environ.update( + PYTHONUNBUFFERED="1", + PYTHONIOENCODING="utf-8", + PIP_NO_DEPS="1", + PIP_NO_BUILD_ISOLATION="1", + PIP_DISABLE_PIP_VERSION_CHECK="1", + PIP_IGNORE_INSTALLED="1" +) + HERE = pathlib.Path(__file__).parent -repos = list(filter(bool, __doc__.splitlines())) +# don't pollute the global state +LINKS = (HERE / ".yarn-links").resolve() +YARN = ["yarn", "--link-folder", LINKS] +PIP = ["python", "-m", "pip"] + +URLS = list(filter(bool, __doc__.splitlines())) def task_lint(): @@ -31,62 +47,82 @@ def task_lint(): # add targets to the docstring to include in the dev build. def task_clone(): """clone all the repos defined in the doc string""" - for repo in repos: - repo, _, branch = repo.rpartition("@") - path = pathlib.Path(repo.rpartition("/")[2]) + for url in URLS: + path = url_to_path(url) yield dict( - name=f"clone_{repo}", - actions=[] if path.exists() else [f"git clone --depth 1 {repo}"], + name=path.name, + actions=[] if path.exists() else [do("git", "clone", "--depth=1", url)], targets=[path / "package.json"] ) -def task_build(): + +def task_setup(): """ensure a working build of live development builds""" - for dep in [repo_to_path(x) / "package.json" for x in repos]: + for repo in map(url_to_path, URLS): yield dict( - name=f"install_{dep}", - file_dep=[dep], - actions=[ - f"""cd {dep.parent} && jlpm --prefer-offline --ignore-optional""" - ], - targets=[dep.parent / "node_modules/.yarn-integrity"] + name=f"{repo.name}:yarn", + file_dep=[repo / "package.json"], + actions=[do(*YARN, cwd=repo)], + targets=yarn_integrity(repo) ) + + setup_py = repo / "setup.py" + + if setup_py.exists(): + yield dict( + name=f"{repo.name}:pip", + file_dep=yarn_integrity(repo), + actions=[ + do(*PIP, "uninstall", "-y", repo.name, cwd=repo), + do(*PIP, "install", "-e", ".", cwd=repo), + do(*PIP, "check") + ] + ) + yield dict( - name=f"build_{dep}", - file_dep=[dep.parent / "node_modules/.yarn-integrity"], + name=f"{repo.name}:build", + file_dep=yarn_integrity(repo), actions=[ - f"""cd {dep.parent} && jlpm build""" + do(*YARN, "build", cwd=repo) ], - targets=list(dep.parent.rglob("packages/*/lib/*.js")) + targets=list(repo.glob("packages/*/lib/*.js")), + **( + dict(task_dep=[f"setup:{repo.name}:pip"]) if setup_py.exists() else {} + ) ) -def task_link_lumino(): - """jupter labextension link changes across the repos""" - # find the changes in the repos relative to master. + +def task_link(): + """link yarn packages across the repos""" # go to the direction and links the packages. - for pkg in pathlib.Path("lumino/packages").glob("*/package.json"): + lumino = [url_to_path(u) for u in URLS if "jupyterlab/lumino" in u][0] + lab = [url_to_path(u) for u in URLS if "jupyterlab/jupyterlab" in u][0] + + for pkg_json in lumino.glob("packages/*/package.json"): + pkg = pkg_json.parent + pkg_data = json.loads(pkg_json.read_text(encoding="utf-8")) + pkg_name = pkg_data["name"] + out_link = LINKS / pkg_data["name"] / "package.json" + in_link = lab / f"node_modules/{pkg_name}/package.json" yield dict( - name=f"link_lumino_{pkg.parent.name}", - file_dep=[ - pkg, - pkg.parent.parent.parent / "node_modules/.yarn-integrity" + name=pkg_name, + file_dep=[*yarn_integrity(lumino), *yarn_integrity(lab), pkg_json], + actions=[ + (doit.tools.create_folder, [LINKS]), + do(*YARN, "link", cwd=pkg) ], - actions=f""" - cd lumino/packages/{pkg.parent.name} \ - && jlpm link - - """.strip().splitlines(),#&& touch ../../build/link.lumino.{pkg.parent.name}.ok - targets=[]# [f"build/link.lumino.{pkg.parent.name}.ok"] + targets=[out_link] ) + yield dict( - name=f"link_lab_{pkg.parent.name}", - file_dep=[],#[f"build/link.lumino.{pkg.parent.name}.ok"], - actions=f""" - cd jupyterlab \ - && jlpm link @lumino/{pkg.parent.name} - - """.strip().splitlines(), #&& touch ../build/link.lab.{pkg.parent.name}.ok - targets=[f"build/link.lab.{pkg.parent.name}.ok"] + name=f"lab:{pkg_name}", + uptodate=[doit.tools.config_changed({ + pkg_name: ( + in_link.exists() and in_link.resolve() == pkg_json.resolve() + ) + })], + file_dep=[out_link], + actions=[do(*YARN, "link", pkg_name, cwd=lab)] ) def task_config(): @@ -110,6 +146,14 @@ def task_config(): # utilities -def repo_to_path(x): +def url_to_path(x): """extract the local checkout name""" return pathlib.Path(x.rpartition("@")[0].rpartition("/")[2]) + +def do(*args, cwd=HERE, **kwargs): + """wrap a CmdAction for consistency""" + return doit.tools.CmdAction(list(args), shell=False, cwd=str(pathlib.Path(cwd))) + +def yarn_integrity(repo): + """the file created after yarn install""" + return [repo / "node_modules/.yarn-integrity"] diff --git a/environment.yml b/environment.yml index 02997687..b58864ea 100644 --- a/environment.yml +++ b/environment.yml @@ -8,12 +8,18 @@ dependencies: - black - doit - git + - jupyter_server + - jupyter-packaging - jupyter-server-proxy + - jupyterlab_server - matplotlib-base + - nbclassic - nodejs>=12,<15,!=13.* - numpy - pip - python>=3.7,<3.10 + - twine - vega_datasets + - wheel - xeus-python - yarn<2 From 53ba41d2f2adf125697d9ec49a54e67439cd0f56 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 23:24:57 -0500 Subject: [PATCH 06/12] fix linting, remote server proxy --- .gitignore | 4 ++ dodo.py | 105 ++++++++++++++++++++++++++++++++---------------- environment.yml | 1 - noxfile.py | 3 +- 4 files changed, 76 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 5dc15646..137aaee7 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,7 @@ venv.bak/ # yarn .yarn-links/ node_modules/ + +# repos +jupyterlab/ +lumino/ diff --git a/dodo.py b/dodo.py index a667b696..83cad26e 100644 --- a/dodo.py +++ b/dodo.py @@ -17,12 +17,13 @@ } os.environ.update( - PYTHONUNBUFFERED="1", - PYTHONIOENCODING="utf-8", - PIP_NO_DEPS="1", - PIP_NO_BUILD_ISOLATION="1", + NODE_OPTS="--max-old-space-size=4096", PIP_DISABLE_PIP_VERSION_CHECK="1", - PIP_IGNORE_INSTALLED="1" + PIP_IGNORE_INSTALLED="1", + PIP_NO_BUILD_ISOLATION="1", + PIP_NO_DEPENDENCIES="1", + PYTHONIOENCODING="utf-8", + PYTHONUNBUFFERED="1", ) HERE = pathlib.Path(__file__).parent @@ -36,12 +37,8 @@ def task_lint(): - all_py = HERE.glob("*.py") - yield dict( - name="py", - file_dep=[*all_py], - actions=[["black", "--quiet", *all_py]] - ) + all_py = [*HERE.glob("*.py")] + yield dict(name="py", file_dep=[*all_py], actions=[do("black", *all_py)]) # add targets to the docstring to include in the dev build. @@ -52,7 +49,7 @@ def task_clone(): yield dict( name=path.name, actions=[] if path.exists() else [do("git", "clone", "--depth=1", url)], - targets=[path / "package.json"] + targets=[path / "package.json"], ) @@ -63,7 +60,7 @@ def task_setup(): name=f"{repo.name}:yarn", file_dep=[repo / "package.json"], actions=[do(*YARN, cwd=repo)], - targets=yarn_integrity(repo) + targets=yarn_integrity(repo), ) setup_py = repo / "setup.py" @@ -75,28 +72,27 @@ def task_setup(): actions=[ do(*PIP, "uninstall", "-y", repo.name, cwd=repo), do(*PIP, "install", "-e", ".", cwd=repo), - do(*PIP, "check") - ] + do(*PIP, "check"), + ], ) yield dict( name=f"{repo.name}:build", file_dep=yarn_integrity(repo), - actions=[ - do(*YARN, "build", cwd=repo) - ], + actions=[do(*YARN, "build", cwd=repo)], targets=list(repo.glob("packages/*/lib/*.js")), - **( - dict(task_dep=[f"setup:{repo.name}:pip"]) if setup_py.exists() else {} - ) + **(dict(task_dep=[f"setup:{repo.name}:pip"]) if setup_py.exists() else {}), ) def task_link(): """link yarn packages across the repos""" # go to the direction and links the packages. - lumino = [url_to_path(u) for u in URLS if "jupyterlab/lumino" in u][0] - lab = [url_to_path(u) for u in URLS if "jupyterlab/jupyterlab" in u][0] + lumino = get_lumino() + lab = get_jupyterlab() + + if not (lumino and lab): + return for pkg_json in lumino.glob("packages/*/package.json"): pkg = pkg_json.parent @@ -107,30 +103,52 @@ def task_link(): yield dict( name=pkg_name, file_dep=[*yarn_integrity(lumino), *yarn_integrity(lab), pkg_json], - actions=[ - (doit.tools.create_folder, [LINKS]), - do(*YARN, "link", cwd=pkg) - ], - targets=[out_link] + actions=[(doit.tools.create_folder, [LINKS]), do(*YARN, "link", cwd=pkg)], + targets=[out_link], ) yield dict( name=f"lab:{pkg_name}", - uptodate=[doit.tools.config_changed({ - pkg_name: ( - in_link.exists() and in_link.resolve() == pkg_json.resolve() + uptodate=[ + doit.tools.config_changed( + { + pkg_name: ( + in_link.exists() and in_link.resolve() == pkg_json.resolve() + ) + } ) - })], + ], file_dep=[out_link], - actions=[do(*YARN, "link", pkg_name, cwd=lab)] + actions=[do(*YARN, "link", pkg_name, cwd=lab)], ) + +def task_rebuild(): + lab = get_jupyterlab() + + if not lab: + return + + dev_mode = lab / "dev_mode" + pkg_static = dev_mode / "static/package.json" + + yield dict( + name="dev:prod", + task_dep=["link"], + actions=[do(*YARN, "build:prod", cwd=dev_mode)], + targets=[pkg_static], + ) + + def task_config(): """merge config""" # hoist the configurations from the existing repos like jupyterlab. # we'll use their start to begin with. - return dict(actions="""cp jupyterlab/binder/start . - cp jupyterlab/binder/jupyter_notebook_config.py .""".splitlines()) + return dict( + actions="""cp jupyterlab/binder/start . + cp jupyterlab/binder/jupyter_notebook_config.py .""".splitlines() + ) + # def task_postBuild(): # """recursively invoke all postBuilds""" @@ -146,14 +164,31 @@ def task_config(): # utilities + def url_to_path(x): """extract the local checkout name""" return pathlib.Path(x.rpartition("@")[0].rpartition("/")[2]) + def do(*args, cwd=HERE, **kwargs): """wrap a CmdAction for consistency""" return doit.tools.CmdAction(list(args), shell=False, cwd=str(pathlib.Path(cwd))) + def yarn_integrity(repo): """the file created after yarn install""" return [repo / "node_modules/.yarn-integrity"] + + +def get_lumino(): + try: + return [url_to_path(u) for u in URLS if "jupyterlab/lumino" in u][0] + except: + print("lumino is not included") + + +def get_jupyterlab(): + try: + return [url_to_path(u) for u in URLS if "jupyterlab/jupyterlab" in u][0] + except: + print("jupyterlab is not included") diff --git a/environment.yml b/environment.yml index b58864ea..838e2dda 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,6 @@ dependencies: - git - jupyter_server - jupyter-packaging - - jupyter-server-proxy - jupyterlab_server - matplotlib-base - nbclassic diff --git a/noxfile.py b/noxfile.py index ae9c6bec..96493548 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,6 +1,7 @@ import nox + + @nox.session(venv_backend="conda") def build(session): session.install("doit") session.run("doit", *session.posargs) - From 446c2175b6f24fbfb8935bd2dc6bf772a323ab11 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 2 Mar 2021 23:28:28 -0500 Subject: [PATCH 07/12] rework task names --- dodo.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/dodo.py b/dodo.py index 83cad26e..b3f05e58 100644 --- a/dodo.py +++ b/dodo.py @@ -56,19 +56,22 @@ def task_clone(): def task_setup(): """ensure a working build of live development builds""" for repo in map(url_to_path, URLS): - yield dict( - name=f"{repo.name}:yarn", - file_dep=[repo / "package.json"], - actions=[do(*YARN, cwd=repo)], - targets=yarn_integrity(repo), - ) + pkg_json = repo / "package.json" + + if pkg_json.exists(): + yield dict( + name=f"yarn:install:{repo.name}", + file_dep=[pkg_json], + actions=[do(*YARN, cwd=repo)], + targets=yarn_integrity(repo), + ) setup_py = repo / "setup.py" if setup_py.exists(): yield dict( - name=f"{repo.name}:pip", - file_dep=yarn_integrity(repo), + name=f"pip:install:{repo.name}", + file_dep=[setup_py] + ([pkg_json] if pkg_json.exists() else []), actions=[ do(*PIP, "uninstall", "-y", repo.name, cwd=repo), do(*PIP, "install", "-e", ".", cwd=repo), @@ -76,13 +79,18 @@ def task_setup(): ], ) - yield dict( - name=f"{repo.name}:build", - file_dep=yarn_integrity(repo), - actions=[do(*YARN, "build", cwd=repo)], - targets=list(repo.glob("packages/*/lib/*.js")), - **(dict(task_dep=[f"setup:{repo.name}:pip"]) if setup_py.exists() else {}), - ) + if pkg_json.exists(): + yield dict( + name=f"yarn:build:{repo.name}", + file_dep=yarn_integrity(repo), + actions=[do(*YARN, "build", cwd=repo)], + targets=list(repo.glob("packages/*/lib/*.js")), + **( + dict(task_dep=[f"setup:pip:install:{repo.name}"]) + if setup_py.exists() + else {} + ), + ) def task_link(): From d5f8617d09171f025671203c430d530adbf3b6d9 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 3 Mar 2021 00:04:16 -0500 Subject: [PATCH 08/12] up to running lab --- dodo.py | 118 +++++++++++++++++++++++++++++++++++++++++++----------- postBuild | 5 ++- 2 files changed, 97 insertions(+), 26 deletions(-) diff --git a/dodo.py b/dodo.py index b3f05e58..1aa9d7c8 100644 --- a/dodo.py +++ b/dodo.py @@ -9,6 +9,9 @@ import pathlib import doit.tools import json +import shutil +import sys +import subprocess DOIT_CONFIG = { "backend": "sqlite3", @@ -33,6 +36,11 @@ YARN = ["yarn", "--link-folder", LINKS] PIP = ["python", "-m", "pip"] +APP_DIR = pathlib.Path(sys.prefix) / "share/jupyter/lab" +APP_STATIC = APP_DIR / "static" +APP_INDEX = APP_STATIC / "index.html" + + URLS = list(filter(bool, __doc__.splitlines())) @@ -69,15 +77,42 @@ def task_setup(): setup_py = repo / "setup.py" if setup_py.exists(): + py_deps = [setup_py] + ([pkg_json] if pkg_json.exists() else []) yield dict( name=f"pip:install:{repo.name}", - file_dep=[setup_py] + ([pkg_json] if pkg_json.exists() else []), + file_dep=py_deps, actions=[ do(*PIP, "uninstall", "-y", repo.name, cwd=repo), do(*PIP, "install", "-e", ".", cwd=repo), do(*PIP, "check"), ], ) + if repo == get_jupyterlab(): + yield dict( + name=f"server:{repo.name}", + file_dep=py_deps, + task_dep=[f"setup:pip:install:{repo.name}"], + actions=sum( + [ + [ + do( + "jupyter", + *app, + "enable", + "--py", + repo.name, + "--sys-prefix", + ), + do("jupyter", *app, "list"), + ] + for app in [ + ["serverextension"], + ["server", "extension"], + ] + ], + [], + ), + ) if pkg_json.exists(): yield dict( @@ -131,44 +166,62 @@ def task_link(): ) -def task_rebuild(): +def task_app(): lab = get_jupyterlab() if not lab: return dev_mode = lab / "dev_mode" - pkg_static = dev_mode / "static/package.json" + dev_static = dev_mode / "static" + dev_index = dev_static / "index.html" yield dict( name="dev:prod", - task_dep=["link"], + file_dep=[ + *LINKS.glob("*/package.json"), + *LINKS.glob("*/*/package.json"), + *sum( + [ + [*repo.glob("packages/*/lib/*.js")] + for repo in map(url_to_path, URLS) + ], + [], + ), + ], actions=[do(*YARN, "build:prod", cwd=dev_mode)], - targets=[pkg_static], + targets=[dev_index], ) - -def task_config(): - """merge config""" - # hoist the configurations from the existing repos like jupyterlab. - # we'll use their start to begin with. - return dict( - actions="""cp jupyterlab/binder/start . - cp jupyterlab/binder/jupyter_notebook_config.py .""".splitlines() + yield dict( + name="dev:copy", + file_dep=[dev_index], + actions=[ + lambda: [shutil.rmtree(APP_DIR, ignore_errors=True), None][-1], + (doit.tools.create_folder, [APP_DIR]), + lambda: [ + shutil.copytree(dev_mode / subdir, APP_DIR / subdir) + for subdir in ["static", "schemas", "templates", "themes"] + ] + and None, + ], + targets=[APP_INDEX], ) -# def task_postBuild(): -# """recursively invoke all postBuilds""" -# for repo in repos: -# for postBuild in [*repo.rglob("postBuild")]: -# yield dict( -# name=repo + str(postBuild), -# file_dep=[postBuild], -# actions=[ -# doit.CmdAction([], shell=False, cwd=postBuild.parent) -# ] -# ) +def task_start(): + if os.environ.get("NOT_ON_BINDER") is None: + print("set the environment variable NOT_ON_BINDER to start") + return + + if get_jupyterlab(): + yield dict( + name="lab", + uptodate=[lambda: False], + file_dep=[APP_INDEX], + actions=[run_lab()], + ) + # utilities @@ -200,3 +253,20 @@ def get_jupyterlab(): return [url_to_path(u) for u in URLS if "jupyterlab/jupyterlab" in u][0] except: print("jupyterlab is not included") + + +def run_lab(extra_args=None): + def lab(): + args = ["jupyter", "lab", "--debug", "--no-browser", *(extra_args or [])] + proc = subprocess.Popen(list(map(str, args)), stdin=subprocess.PIPE) + + try: + proc.wait() + except KeyboardInterrupt: + proc.terminate() + proc.communicate(b"y\n") + + proc.wait() + return True + + return doit.tools.PythonInteractiveAction(lab) diff --git a/postBuild b/postBuild index a4efc6cf..52a9635d 100644 --- a/postBuild +++ b/postBuild @@ -1,5 +1,6 @@ +#!/usr/bin/env bash doit list --all --status -doit +doit || doit || doit -cd jupyterlab && pip install -e . && jlpm && jlpm build +doit list --all --status From 4ba662555fb83e244234fe8b55462ab7f80d4e43 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 3 Mar 2021 00:17:05 -0500 Subject: [PATCH 09/12] add some logging and timing --- dodo.py | 2 +- postBuild | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100644 => 100755 postBuild diff --git a/dodo.py b/dodo.py index 1aa9d7c8..634d767a 100644 --- a/dodo.py +++ b/dodo.py @@ -189,7 +189,7 @@ def task_app(): [], ), ], - actions=[do(*YARN, "build:prod", cwd=dev_mode)], + actions=[do(*YARN, "build", cwd=dev_mode)], targets=[dev_index], ) diff --git a/postBuild b/postBuild old mode 100644 new mode 100755 index 52a9635d..16b1ba62 --- a/postBuild +++ b/postBuild @@ -1,6 +1,9 @@ #!/usr/bin/env bash doit list --all --status -doit || doit || doit +time doit clone | tee doit.clone.log +time doit setup | tee doit.setup.log +time doit link | tee doit.link.log +time doit app | tee doit.app.log doit list --all --status From bef6f28f5906464e60e8c7ff40fe9a989f9f9a17 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 3 Mar 2021 01:16:38 -0500 Subject: [PATCH 10/12] move repo defs to other file --- dodo.py | 122 ++++++++++++++++++++++-------------------------- environment.yml | 1 + repos.yml | 10 ++++ 3 files changed, 66 insertions(+), 67 deletions(-) create mode 100644 repos.yml diff --git a/dodo.py b/dodo.py index 634d767a..0c112a3d 100644 --- a/dodo.py +++ b/dodo.py @@ -1,10 +1,4 @@ -#!/usr/bin/env doit -f -"""https://github.com/jupyterlab/jupyterlab@408f30f -https://github.com/jupyterlab/lumino@09aec10 -""" -# change the urls above to link different jupyter references. - - +# change the URLs in `./repos.yml` import os import pathlib import doit.tools @@ -12,6 +6,7 @@ import shutil import sys import subprocess +from yaml import safe_load DOIT_CONFIG = { "backend": "sqlite3", @@ -41,7 +36,8 @@ APP_INDEX = APP_STATIC / "index.html" -URLS = list(filter(bool, __doc__.splitlines())) +COMMITS = safe_load((HERE / "repos.yml").read_text())["repos"] +REPOS = {name: HERE / name for name in COMMITS} def task_lint(): @@ -52,24 +48,46 @@ def task_lint(): # add targets to the docstring to include in the dev build. def task_clone(): """clone all the repos defined in the doc string""" - for url in URLS: - path = url_to_path(url) + for name, spec in COMMITS.items(): + path = HERE / name + config = path / ".git/config" + head = path / ".git/HEAD" + + yield dict( + name=f"init:{name}", + uptodate=[doit.tools.config_changed(spec)], + actions=[] + if path.exists() + else [ + (doit.tools.create_folder, [path]), + do("git", "init", cwd=path), + do("git", "remote", "add", "origin", spec["origin"], cwd=path), + ], + targets=[config], + ) + yield dict( - name=path.name, - actions=[] if path.exists() else [do("git", "clone", "--depth=1", url)], - targets=[path / "package.json"], + name=f"""fetch:{name}:{spec["ref"]}""", + file_dep=[config], + actions=[ + do("git", "fetch", "origin", spec["ref"], cwd=path), + do("git", "checkout", spec["commit"], cwd=path), + do("git", "log", "--name-status", "HEAD^..HEAD", cwd=path), + ], + targets=[head], ) def task_setup(): """ensure a working build of live development builds""" - for repo in map(url_to_path, URLS): + for repo in REPOS.values(): + head = repo / ".git/HEAD" pkg_json = repo / "package.json" if pkg_json.exists(): yield dict( name=f"yarn:install:{repo.name}", - file_dep=[pkg_json], + file_dep=[pkg_json, head], actions=[do(*YARN, cwd=repo)], targets=yarn_integrity(repo), ) @@ -77,7 +95,9 @@ def task_setup(): setup_py = repo / "setup.py" if setup_py.exists(): - py_deps = [setup_py] + ([pkg_json] if pkg_json.exists() else []) + py_deps = [head, setup_py] + ( + yarn_integrity(repo) if pkg_json.exists() else [] + ) yield dict( name=f"pip:install:{repo.name}", file_dep=py_deps, @@ -87,31 +107,12 @@ def task_setup(): do(*PIP, "check"), ], ) - if repo == get_jupyterlab(): + if repo == REPOS.get("jupyterlab"): yield dict( name=f"server:{repo.name}", file_dep=py_deps, task_dep=[f"setup:pip:install:{repo.name}"], - actions=sum( - [ - [ - do( - "jupyter", - *app, - "enable", - "--py", - repo.name, - "--sys-prefix", - ), - do("jupyter", *app, "list"), - ] - for app in [ - ["serverextension"], - ["server", "extension"], - ] - ], - [], - ), + actions=server_extensions(repo), ) if pkg_json.exists(): @@ -131,8 +132,8 @@ def task_setup(): def task_link(): """link yarn packages across the repos""" # go to the direction and links the packages. - lumino = get_lumino() - lab = get_jupyterlab() + lumino = REPOS.get("lumino") + lab = REPOS.get("jupyterlab") if not (lumino and lab): return @@ -167,7 +168,7 @@ def task_link(): def task_app(): - lab = get_jupyterlab() + lab = REPOS.get("jupyterlab") if not lab: return @@ -177,15 +178,12 @@ def task_app(): dev_index = dev_static / "index.html" yield dict( - name="dev:prod", + name="build", file_dep=[ *LINKS.glob("*/package.json"), *LINKS.glob("*/*/package.json"), *sum( - [ - [*repo.glob("packages/*/lib/*.js")] - for repo in map(url_to_path, URLS) - ], + [[*repo.glob("packages/*/lib/*.js")] for repo in REPOS.values()], [], ), ], @@ -194,7 +192,7 @@ def task_app(): ) yield dict( - name="dev:copy", + name="deploy", file_dep=[dev_index], actions=[ lambda: [shutil.rmtree(APP_DIR, ignore_errors=True), None][-1], @@ -214,7 +212,7 @@ def task_start(): print("set the environment variable NOT_ON_BINDER to start") return - if get_jupyterlab(): + if "jupyterlab" in REPOS: yield dict( name="lab", uptodate=[lambda: False], @@ -226,11 +224,6 @@ def task_start(): # utilities -def url_to_path(x): - """extract the local checkout name""" - return pathlib.Path(x.rpartition("@")[0].rpartition("/")[2]) - - def do(*args, cwd=HERE, **kwargs): """wrap a CmdAction for consistency""" return doit.tools.CmdAction(list(args), shell=False, cwd=str(pathlib.Path(cwd))) @@ -241,24 +234,19 @@ def yarn_integrity(repo): return [repo / "node_modules/.yarn-integrity"] -def get_lumino(): - try: - return [url_to_path(u) for u in URLS if "jupyterlab/lumino" in u][0] - except: - print("lumino is not included") - - -def get_jupyterlab(): - try: - return [url_to_path(u) for u in URLS if "jupyterlab/jupyterlab" in u][0] - except: - print("jupyterlab is not included") +def server_extensions(repo): + enable = ["enable", "--py", repo.name, "--sys-prefix"] + apps = ["serverextension"], ["server", "extension"] + return sum( + [[do("jupyter", *app, *enable), do("jupyter", *app, "list")] for app in apps], + [], + ) -def run_lab(extra_args=None): +def run_lab(): def lab(): - args = ["jupyter", "lab", "--debug", "--no-browser", *(extra_args or [])] - proc = subprocess.Popen(list(map(str, args)), stdin=subprocess.PIPE) + args = ["jupyter", "lab", "--debug", "--no-browser"] + proc = subprocess.Popen(args, stdin=subprocess.PIPE) try: proc.wait() diff --git a/environment.yml b/environment.yml index 838e2dda..a747086f 100644 --- a/environment.yml +++ b/environment.yml @@ -17,6 +17,7 @@ dependencies: - numpy - pip - python>=3.7,<3.10 + - pyyaml - twine - vega_datasets - wheel diff --git a/repos.yml b/repos.yml new file mode 100644 index 00000000..b3c38c25 --- /dev/null +++ b/repos.yml @@ -0,0 +1,10 @@ +# these are some repos we want to check out and test against: +repos: + jupyterlab: + origin: https://github.com/jupyterlab/jupyterlab + ref: pull/9622/head + commit: "408f30f" + lumino: + origin: https://github.com/jupyterlab/lumino + ref: pull/149/head + commit: "09aec10" From 96e3b0de0307d1688204e34430b553b454dcc5b3 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 3 Mar 2021 08:56:59 -0500 Subject: [PATCH 11/12] rework paths, commit specs --- .gitignore | 3 +- dodo.py | 89 ++++++++++++++++++++++++++++++------------------------ postBuild | 8 ++--- repos.yml | 19 +++++++++--- 4 files changed, 68 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index 137aaee7..378862c0 100644 --- a/.gitignore +++ b/.gitignore @@ -111,5 +111,4 @@ venv.bak/ node_modules/ # repos -jupyterlab/ -lumino/ +repos/ diff --git a/dodo.py b/dodo.py index 0c112a3d..b7520cd2 100644 --- a/dodo.py +++ b/dodo.py @@ -27,7 +27,7 @@ HERE = pathlib.Path(__file__).parent # don't pollute the global state -LINKS = (HERE / ".yarn-links").resolve() +LINKS = (HERE / "repos/.yarn-links").resolve() YARN = ["yarn", "--link-folder", LINKS] PIP = ["python", "-m", "pip"] @@ -36,8 +36,9 @@ APP_INDEX = APP_STATIC / "index.html" -COMMITS = safe_load((HERE / "repos.yml").read_text())["repos"] -REPOS = {name: HERE / name for name in COMMITS} +REPOS_YML = HERE / "repos.yml" +REPOS = safe_load(REPOS_YML.read_text())["repos"] +PATHS = {name: HERE / "repos" / name for name in REPOS} def task_lint(): @@ -48,14 +49,16 @@ def task_lint(): # add targets to the docstring to include in the dev build. def task_clone(): """clone all the repos defined in the doc string""" - for name, spec in COMMITS.items(): - path = HERE / name + for name, spec in REPOS.items(): + path = PATHS[name] config = path / ".git/config" head = path / ".git/HEAD" + uptodate = doit.tools.config_changed({name: spec}) + yield dict( name=f"init:{name}", - uptodate=[doit.tools.config_changed(spec)], + file_dep=[REPOS_YML], actions=[] if path.exists() else [ @@ -66,63 +69,69 @@ def task_clone(): targets=[config], ) - yield dict( - name=f"""fetch:{name}:{spec["ref"]}""", - file_dep=[config], - actions=[ - do("git", "fetch", "origin", spec["ref"], cwd=path), - do("git", "checkout", spec["commit"], cwd=path), - do("git", "log", "--name-status", "HEAD^..HEAD", cwd=path), - ], - targets=[head], - ) + for i, ref in enumerate(spec["refs"]): + task_dep = [] + actions = [do("git", "fetch", "origin", ref["ref"], cwd=path)] + if i: + task_dep += [f"""clone:fetch:{name}:{i-1}:{ref["refs"][i - 1]}"""] + actions += [do("git", "merge", f"""origin/{ref["commit"]}""")] + else: + actions += [do("git", "checkout", "-f", ref["commit"], cwd=path)] + + yield dict( + name=f"""fetch:{name}:{i}:{ref["ref"]}:{ref["commit"]}""", + file_dep=[config], + targets=[head], + task_dep=task_dep, + actions=actions, + ) def task_setup(): """ensure a working build of live development builds""" - for repo in REPOS.values(): - head = repo / ".git/HEAD" - pkg_json = repo / "package.json" + for path in PATHS.values(): + head = path / ".git/HEAD" + pkg_json = path / "package.json" if pkg_json.exists(): yield dict( - name=f"yarn:install:{repo.name}", + name=f"yarn:install:{path.name}", file_dep=[pkg_json, head], - actions=[do(*YARN, cwd=repo)], - targets=yarn_integrity(repo), + actions=[do(*YARN, cwd=path)], + targets=yarn_integrity(path), ) - setup_py = repo / "setup.py" + setup_py = path / "setup.py" if setup_py.exists(): py_deps = [head, setup_py] + ( - yarn_integrity(repo) if pkg_json.exists() else [] + yarn_integrity(path) if pkg_json.exists() else [] ) yield dict( - name=f"pip:install:{repo.name}", + name=f"pip:install:{path.name}", file_dep=py_deps, actions=[ - do(*PIP, "uninstall", "-y", repo.name, cwd=repo), - do(*PIP, "install", "-e", ".", cwd=repo), + do(*PIP, "uninstall", "-y", path.name, cwd=path), + do(*PIP, "install", "-e", ".", cwd=path), do(*PIP, "check"), ], ) - if repo == REPOS.get("jupyterlab"): + if path == REPOS.get("jupyterlab"): yield dict( - name=f"server:{repo.name}", + name=f"server:{path.name}", file_dep=py_deps, - task_dep=[f"setup:pip:install:{repo.name}"], - actions=server_extensions(repo), + task_dep=[f"setup:pip:install:{path.name}"], + actions=server_extensions(path), ) if pkg_json.exists(): yield dict( - name=f"yarn:build:{repo.name}", - file_dep=yarn_integrity(repo), - actions=[do(*YARN, "build", cwd=repo)], - targets=list(repo.glob("packages/*/lib/*.js")), + name=f"yarn:build:{path.name}", + file_dep=yarn_integrity(path), + actions=[do(*YARN, "build", cwd=path)], + targets=list(path.glob("packages/*/lib/*.js")), **( - dict(task_dep=[f"setup:pip:install:{repo.name}"]) + dict(task_dep=[f"setup:pip:install:{path.name}"]) if setup_py.exists() else {} ), @@ -132,8 +141,8 @@ def task_setup(): def task_link(): """link yarn packages across the repos""" # go to the direction and links the packages. - lumino = REPOS.get("lumino") - lab = REPOS.get("jupyterlab") + lumino = PATHS.get("lumino") + lab = PATHS.get("jupyterlab") if not (lumino and lab): return @@ -168,7 +177,7 @@ def task_link(): def task_app(): - lab = REPOS.get("jupyterlab") + lab = PATHS.get("jupyterlab") if not lab: return @@ -183,7 +192,7 @@ def task_app(): *LINKS.glob("*/package.json"), *LINKS.glob("*/*/package.json"), *sum( - [[*repo.glob("packages/*/lib/*.js")] for repo in REPOS.values()], + [[*repo.glob("packages/*/lib/*.js")] for repo in PATHS.values()], [], ), ], diff --git a/postBuild b/postBuild index 16b1ba62..6b7d86d7 100755 --- a/postBuild +++ b/postBuild @@ -1,9 +1,9 @@ #!/usr/bin/env bash doit list --all --status -time doit clone | tee doit.clone.log -time doit setup | tee doit.setup.log -time doit link | tee doit.link.log -time doit app | tee doit.app.log +time doit clone |& tee doit.clone.log +time doit setup |& tee doit.setup.log +time doit link |& tee doit.link.log +time doit app |& tee doit.app.log doit list --all --status diff --git a/repos.yml b/repos.yml index b3c38c25..c5159ec9 100644 --- a/repos.yml +++ b/repos.yml @@ -1,10 +1,19 @@ -# these are some repos we want to check out and test against: +# these are some repos we want to check out and test against. +# these aren't submodules, so that they can be easily updated from the web UI repos: + # a local symbolic name, also where packages will be checked out in `packages` jupyterlab: + # the public remote from which to start origin: https://github.com/jupyterlab/jupyterlab - ref: pull/9622/head - commit: "408f30f" + # an ordered list of refs to attempt merging: must be fast-forward + refs: + - # the upstream ref, found with `git ls-remote` + ref: pull/9622/head + # the specific committish, available in the ref + commit: "408f30f" + lumino: origin: https://github.com/jupyterlab/lumino - ref: pull/149/head - commit: "09aec10" + refs: + - ref: pull/149/head + commit: "09aec10" From 22af71933156c0a404cba3e4e8464aa5f311303c Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Wed, 3 Mar 2021 09:03:10 -0500 Subject: [PATCH 12/12] some docs --- dodo.py | 93 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/dodo.py b/dodo.py index b7520cd2..5c89638e 100644 --- a/dodo.py +++ b/dodo.py @@ -31,9 +31,9 @@ YARN = ["yarn", "--link-folder", LINKS] PIP = ["python", "-m", "pip"] -APP_DIR = pathlib.Path(sys.prefix) / "share/jupyter/lab" -APP_STATIC = APP_DIR / "static" -APP_INDEX = APP_STATIC / "index.html" +LAB_APP_DIR = pathlib.Path(sys.prefix) / "share/jupyter/lab" +LAB_APP_STATIC = LAB_APP_DIR / "static" +LAB_APP_INDEX = LAB_APP_STATIC / "index.html" REPOS_YML = HERE / "repos.yml" @@ -48,7 +48,7 @@ def task_lint(): # add targets to the docstring to include in the dev build. def task_clone(): - """clone all the repos defined in the doc string""" + """clone all the repos defined in `repos.yml`""" for name, spec in REPOS.items(): path = PATHS[name] config = path / ".git/config" @@ -88,7 +88,7 @@ def task_clone(): def task_setup(): - """ensure a working build of live development builds""" + """ensure a working build of repos""" for path in PATHS.values(): head = path / ".git/HEAD" pkg_json = path / "package.json" @@ -177,56 +177,56 @@ def task_link(): def task_app(): + """rebuild apps with live modifications""" lab = PATHS.get("jupyterlab") - if not lab: - return + if lab: + dev_mode = lab / "dev_mode" + dev_static = dev_mode / "static" + dev_index = dev_static / "index.html" - dev_mode = lab / "dev_mode" - dev_static = dev_mode / "static" - dev_index = dev_static / "index.html" - - yield dict( - name="build", - file_dep=[ - *LINKS.glob("*/package.json"), - *LINKS.glob("*/*/package.json"), - *sum( - [[*repo.glob("packages/*/lib/*.js")] for repo in PATHS.values()], - [], - ), - ], - actions=[do(*YARN, "build", cwd=dev_mode)], - targets=[dev_index], - ) + yield dict( + name="build", + file_dep=[ + *LINKS.glob("*/package.json"), + *LINKS.glob("*/*/package.json"), + *sum( + [[*repo.glob("packages/*/lib/*.js")] for repo in PATHS.values()], + [], + ), + ], + actions=[do(*YARN, "build", cwd=dev_mode)], + targets=[dev_index], + ) - yield dict( - name="deploy", - file_dep=[dev_index], - actions=[ - lambda: [shutil.rmtree(APP_DIR, ignore_errors=True), None][-1], - (doit.tools.create_folder, [APP_DIR]), - lambda: [ - shutil.copytree(dev_mode / subdir, APP_DIR / subdir) - for subdir in ["static", "schemas", "templates", "themes"] - ] - and None, - ], - targets=[APP_INDEX], - ) + yield dict( + name="deploy", + file_dep=[dev_index], + actions=[ + lambda: [shutil.rmtree(LAB_APP_DIR, ignore_errors=True), None][-1], + (doit.tools.create_folder, [LAB_APP_DIR]), + lambda: [ + shutil.copytree(dev_mode / subdir, LAB_APP_DIR / subdir) + for subdir in ["static", "schemas", "templates", "themes"] + ] + and None, + ], + targets=[LAB_APP_INDEX], + ) def task_start(): + """start applications""" if os.environ.get("NOT_ON_BINDER") is None: print("set the environment variable NOT_ON_BINDER to start") return if "jupyterlab" in REPOS: yield dict( - name="lab", + name="jupyterlab", uptodate=[lambda: False], - file_dep=[APP_INDEX], - actions=[run_lab()], + file_dep=[LAB_APP_INDEX], + actions=[run_jupyterlab()], ) @@ -239,11 +239,12 @@ def do(*args, cwd=HERE, **kwargs): def yarn_integrity(repo): - """the file created after yarn install""" + """get the file created after yarn install""" return [repo / "node_modules/.yarn-integrity"] def server_extensions(repo): + """enable server( )extensions in a repo""" enable = ["enable", "--py", repo.name, "--sys-prefix"] apps = ["serverextension"], ["server", "extension"] return sum( @@ -252,8 +253,10 @@ def server_extensions(repo): ) -def run_lab(): - def lab(): +def run_jupyterlab(): + """start a jupyterlab application""" + + def jupyterlab(): args = ["jupyter", "lab", "--debug", "--no-browser"] proc = subprocess.Popen(args, stdin=subprocess.PIPE) @@ -266,4 +269,4 @@ def lab(): proc.wait() return True - return doit.tools.PythonInteractiveAction(lab) + return doit.tools.PythonInteractiveAction(jupyterlab)