rewrite Cronjob build triggering logic in python; read from influxdb

.github/dependabot.yml

@@ -26,3 +26,8 @@ updates:
       deps:
         patterns:
           - "*"
+  - package-ecosystem: "pip"
+    # This will update cronjob_scripts/requirements.txt
+    directory: "/cronjob_scripts"
+    schedule:
+      interval: "daily"

.github/workflows/cronjob.yml

@@ -20,10 +20,8 @@ jobs:
 
       - name: Trigger Package Build
         id: find_crate
-        run: |
-          set -euo pipefail
-          touch .env
-          ./trigger-package-build.sh
+        run: make trigger-all
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
           CRATE_CHECK_LIMIT: 10

.github/workflows/selfbuild.yml

@@ -46,7 +46,7 @@ jobs:
           TARGET_ARCH: ${{ matrix.target_arch }}
         run: |
           set -euxo pipefail
-          touch .env
+
           VERSION="$(
             curl \
               --user-agent "cargo-quickinstall build pipeline ([email protected])" \

.github/workflows/weekly.yml

@@ -2,6 +2,7 @@ name: Weekly Recheck
 
 concurrency:
   group: trigger-package-build
+  cancel-in-progress: true
 
 # Run every sunday, to recalculate excludes
 on:
@@ -21,11 +22,9 @@ jobs:
           ssh-key: ${{ secrets.CRONJOB_DEPLOY_KEY }}
           persist-credentials: true
 
-      - name: recheck excludes for all archetectures
+      - name: recheck cargo-quickinstall and some random crates on all architectures
         id: find_crate
-        run: |
-          set -euo pipefail
-          touch .env
-          make recheck
+        run: make recheck
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}

.gitignore

@@ -3,3 +3,7 @@
 .idea
 .vagrant
 \.DS_Store
+__pycache__
+# For now, we rely on scripts/requirements.txt to act as our lockfile, because dependabot doesn't support uv yet.
+cronjob_scripts/uv.lock
+cronjob_scripts/.python-deps-updated.timestamp

Makefile

@@ -2,38 +2,49 @@
 publish: release ## alias for `make release`
 
 .PHONY: release
-release: ## Publish a new release
+release: cronjob_scripts/.python-deps-updated.timestamp ## Publish a new release
 	(cd cargo-quickinstall/ && cargo release patch --execute --no-push)
 	git push origin HEAD:release --tags
 	make recheck
 
+cronjob_scripts/requirements.txt: cronjob_scripts/pyproject.toml ## Compile the python dependencies from pyproject.toml into requirements.txt
+	(cd cronjob_scripts && uv pip compile pyproject.toml --python-version=3.8 --output-file requirements.txt)
+
+# install python dependencies and then record that we've done so so we don't do it again
+# WARNING: this will mess with whatever python venv you happen to be in.
+# this is run on the github actions runner so we can't use uv
+cronjob_scripts/.python-deps-updated.timestamp: cronjob_scripts/requirements.txt
+	python --version
+	pip install -r cronjob_scripts/requirements.txt
+	touch cronjob_scripts/.python-deps-updated.timestamp
+
 .PHONY: windows
-windows: ## trigger a windows build
-	RECHECK=1 TARGET_ARCH=x86_64-pc-windows-msvc ./trigger-package-build.sh
+windows: cronjob_scripts/.python-deps-updated.timestamp ## trigger a windows build
+	RECHECK=1 TARGET_ARCH=x86_64-pc-windows-msvc python cronjob_scripts/trigger-package-build.py
 
 .PHONY: mac
-mac: ## trigger a mac build
-	RECHECK=1 TARGET_ARCH=x86_64-apple-darwin ./trigger-package-build.sh
+mac: cronjob_scripts/.python-deps-updated.timestamp ## trigger a mac build
+	RECHECK=1 TARGET_ARCH=x86_64-apple-darwin python cronjob_scripts/trigger-package-build.py
 
 .PHONY: m1
-m1: ## trigger a mac m1 build
-	RECHECK=1 TARGET_ARCH=aarch64-apple-darwin ./trigger-package-build.sh
+m1: cronjob_scripts/.python-deps-updated.timestamp ## trigger a mac m1 build
+	RECHECK=1 TARGET_ARCH=aarch64-apple-darwin python cronjob_scripts/trigger-package-build.py
 
 .PHONY: linux
-linux: ## trigger a linux build
-	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-gnu ./trigger-package-build.sh
+linux: cronjob_scripts/.python-deps-updated.timestamp ## trigger a linux build
+	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-gnu python cronjob_scripts/trigger-package-build.py
 
 .PHONY: linux-musl
-linux-musl: ## trigger a musl libc-based linux build
-	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-musl ./trigger-package-build.sh
-
-.PHONY: exclude
-exclude: ## recompute excludes, but don't push anywhere (see /tmp/cargo-quickinstall-* for repos)
-	REEXCLUDE=1 ./trigger-package-build.sh
+linux-musl: cronjob_scripts/.python-deps-updated.timestamp ## trigger a musl libc-based linux build
+	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-musl python cronjob_scripts/trigger-package-build.py
 
 .PHONY: recheck
-recheck: ## recompute excludes and start from the top
-	RECHECK=1 ./trigger-package-build.sh
+recheck: cronjob_scripts/.python-deps-updated.timestamp ## build ourself and some random packages on all arches
+	RECHECK=1 TARGET_ARCH=all python cronjob_scripts/trigger-package-build.py
+
+.PHONY: trigger-all
+trigger-all: cronjob_scripts/.python-deps-updated.timestamp ## build some random packages on all arches
+	TARGET_ARCH=all python cronjob_scripts/trigger-package-build.py
 
 .PHONY: help
 help: ## Display this help screen

README.md

@@ -45,7 +45,7 @@ Edit the command however you need, and paste it into your CI pipeline.
 
 ## Supported targets
 
-Check [supported-targets](/supported-targets) for lists of targets quickinstall
+Check [supported-targets](/cronjob_scripts/trigger-package-build.py) for lists of targets quickinstall
 can build for.
 
 ## Limitations

cronjob_scripts/.python-version

@@ -0,0 +1 @@
+3.10

cronjob_scripts/README.md

@@ -0,0 +1,7 @@
+# Scripts
+
+This folder contains python scripts for use in cargo-quickinstall github actions for triggering builds.
+
+This code has some quite heavy python dependencies and strong assumptions about running on unix, so we should not use it on the github actions runners that actually do the package building.
+
+TODO: make a build_scripts/ folder and move all of the package building scripts into there (converting to python as desired).

cronjob_scripts/checkout_worktree.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import subprocess
+
+
+def checkout_worktree_for_arch(target_arch: str):
+    """
+    Checkout a git worktree for the given target_arch, in /tmp.
+
+    This is required for reading the exclude files.
+
+    This is lifted directly from the old trigger-package-build.sh script, and is only expected to
+    work on linux/macos with dash/bash.
+    """
+    worktree_path = f"/tmp/cargo-quickinstall-{target_arch}"
+    bash_script = f"""
+        set -eux
+
+        rm -rf {worktree_path}
+        git worktree remove -f {worktree_path} || true
+        git branch -D "trigger/{target_arch}" || true
+
+        git worktree add --force --force {worktree_path}
+        cd {worktree_path}
+
+        if git fetch origin "trigger/{target_arch}"; then
+            git checkout "origin/trigger/{target_arch}" -B "trigger/{target_arch}"
+        elif git checkout "trigger/{target_arch}"; then
+            # pass
+            true
+        else
+            # New branch with no history. Credit: https://stackoverflow.com/a/13969482
+            git checkout --orphan "trigger/{target_arch}"
+            git rm --cached -r . || true
+            git commit -m "Initial Commit" --allow-empty
+            git push origin "trigger/{target_arch}"
+        fi
+    """
+    subprocess.run(bash_script, shell=True, check=True, text=True)
+    return worktree_path
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <target_arch>")
+        sys.exit(1)
+
+    worktree_path = checkout_worktree_for_arch(sys.argv[1])
+    print(f"checked out to {worktree_path}")

cronjob_scripts/get_latest_version.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import json
+from typing import TypedDict
+import requests
+
+import semver
+
+
+def get_index_url(crate: str):
+    """
+    Packages with 1 character names are placed in a directory named 1.
+    Packages with 2 character names are placed in a directory named 2.
+    Packages with 3 character names are placed in the directory 3/{first-character} where {first-character} is the first character of the package name.
+    All other packages are stored in directories named {first-two}/{second-two} where the top directory is the first two characters of the package name, and the next subdirectory is the third and fourth characters of the package name. For example, cargo would be stored in a file named ca/rg/cargo.
+    -- https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
+    """
+    if len(crate) == 1:
+        return f"https://index.crates.io/1/{crate}"
+    elif len(crate) == 2:
+        return f"https://index.crates.io/2/{crate}"
+    elif len(crate) == 3:
+        return f"https://index.crates.io/3/{crate[0]}/{crate}"
+    else:
+        return f"https://index.crates.io/{crate[:2]}/{crate[2:4]}/{crate}"
+
+
+class CrateVersionDict(TypedDict):
+    name: str
+    vers: str
+    features: dict[str, list[str]]
+
+
+def get_latest_version(crate: str) -> CrateVersionDict:
+    """
+    Calls the crates.io index API to get the latest version of the given crate.
+
+    There is no rate limit on this api, so we can call it as much as we like.
+
+    Note that the return type also includes all of the fields from the jsonschema at
+    https://doc.rust-lang.org/cargo/reference/registry-index.html#json-schema
+    but I can't be bothered typing them right now.
+    """
+    url = get_index_url(crate)
+
+    # FIXME: handle 404s
+    response = requests.get(url)
+    versions = [json.loads(line) for line in response.text.splitlines()]
+    unyanked_versions = [
+        version
+        for version in versions
+        if not version["yanked"]
+        and not semver.VersionInfo.parse(version["vers"]).prerelease
+    ]
+
+    # FIXME: check that rust actually agrees with semver when it comes to pre-release versions etc.
+    unyanked_versions.sort(key=lambda v: semver.VersionInfo.parse(v["vers"]))
+
+    return unyanked_versions[-1]
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <crate>")
+        sys.exit(1)
+
+    print(get_latest_version(sys.argv[1])["vers"])

cronjob_scripts/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "cronjob-scripts"
+version = "0.0.0"
+description = "Scripts for use in cargo-quickinstall github actions"
+readme = "README.md"
+# This is what ubuntu-latest has installed by default
+requires-python = ">=3.8"
+dependencies = [
+    "influxdb3-python>=0.8.0",
+    "requests>=2.32.3",
+    "semver>=3.0.2",
+]

cronjob_scripts/requirements.txt

@@ -0,0 +1,34 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --python-version=3.8 --output-file requirements.txt
+certifi==2024.8.30
+    # via
+    #   influxdb3-python
+    #   requests
+charset-normalizer==3.3.2
+    # via requests
+idna==3.8
+    # via requests
+influxdb3-python==0.8.0
+    # via cronjob-scripts (pyproject.toml)
+numpy==1.24.4
+    # via pyarrow
+pyarrow==17.0.0
+    # via influxdb3-python
+python-dateutil==2.9.0.post0
+    # via influxdb3-python
+reactivex==4.0.4
+    # via influxdb3-python
+requests==2.32.3
+    # via cronjob-scripts (pyproject.toml)
+semver==3.0.2
+    # via cronjob-scripts (pyproject.toml)
+setuptools==74.1.2
+    # via influxdb3-python
+six==1.16.0
+    # via python-dateutil
+typing-extensions==4.12.2
+    # via reactivex
+urllib3==2.2.2
+    # via
+    #   influxdb3-python
+    #   requests

cronjob_scripts/stats.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import os
+
+from influxdb_client_3 import InfluxDBClient3
+
+TOKEN = os.environ.get("INFLUXDB_TOKEN")
+ORG = "cargo-bins"
+HOST = "https://us-east-1-1.aws.cloud2.influxdata.com"
+DATABASE = "cargo-quickinstall"
+
+
+def get_stats(period: str, arch: str | None):
+    client = InfluxDBClient3(host=HOST, token=TOKEN, org=ORG, database=DATABASE)
+
+    query = """
+        SELECT crate
+        FROM "counts"
+        WHERE
+            time >= now() - $period::interval and time <= now()
+            and $arch is null or arch = $arch
+        GROUP BY crate
+        ORDER BY crate
+    """
+
+    # FIXME: pyarrow.Table doesn't have types: https://github.com/apache/arrow/issues/32609
+    table = client.query(
+        query=query,
+        language="sql",
+        query_parameters={
+            "period": f"{period}",
+            "arch": arch,
+        },
+    )
+
+    return table
+
+
+def get_requested_crates(period: str, arch: str | None) -> list[str]:
+    table = get_stats(period=period, arch=arch)
+    return [str(crate) for crate in table["crate"]]
+
+
+if __name__ == "__main__":
+    table = get_stats(period="1 day", arch=None)
+    for crate in table["crate"]:
+        print(crate)