rewrite Cronjob build triggering logic in python; read from influxdb

.github/dependabot.yml

@@ -26,3 +26,8 @@ updates:
       deps:
         patterns:
           - "*"
+  - package-ecosystem: "pip"
+    # This will update cronjob_scripts/requirements.txt
+    directory: "/cronjob_scripts"
+    schedule:
+      interval: "daily"

.github/workflows/cronjob.yml

@@ -20,10 +20,8 @@ jobs:
 
       - name: Trigger Package Build
         id: find_crate
-        run: |
-          set -euo pipefail
-          touch .env
-          ./trigger-package-build.sh
+        run: make trigger-all
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
           CRATE_CHECK_LIMIT: 10

.github/workflows/selfbuild.yml

@@ -46,7 +46,7 @@ jobs:
           TARGET_ARCH: ${{ matrix.target_arch }}
         run: |
           set -euxo pipefail
-          touch .env
+
           VERSION="$(
             curl \
               --user-agent "cargo-quickinstall build pipeline ([email protected])" \

.github/workflows/test.yml

@@ -52,9 +52,12 @@ jobs:
       - name: Fetch tags required for testing
         run: git fetch --tags
 
-      - name: Run test
+      - name: Run cargo test
         run: cargo test
 
+      - name: Test python cronjob_scripts
+        run: make test-cronjob-scripts
+
   e2etest:
     runs-on: ${{ matrix.os }}
     strategy:

.github/workflows/weekly.yml

@@ -2,6 +2,7 @@ name: Weekly Recheck
 
 concurrency:
   group: trigger-package-build
+  cancel-in-progress: true
 
 # Run every sunday, to recalculate excludes
 on:
@@ -21,11 +22,9 @@ jobs:
           ssh-key: ${{ secrets.CRONJOB_DEPLOY_KEY }}
           persist-credentials: true
 
-      - name: recheck excludes for all archetectures
+      - name: recheck cargo-quickinstall and some random crates on all architectures
         id: find_crate
-        run: |
-          set -euo pipefail
-          touch .env
-          make recheck
+        run: make recheck
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}

.gitignore

@@ -3,3 +3,7 @@
 .idea
 .vagrant
 \.DS_Store
+__pycache__
+# For now, we rely on scripts/requirements.txt to act as our lockfile, because dependabot doesn't support uv yet.
+cronjob_scripts/uv.lock
+cronjob_scripts/.python-deps-updated.timestamp

Makefile

@@ -2,38 +2,53 @@
 publish: release ## alias for `make release`
 
 .PHONY: release
-release: ## Publish a new release
+release: cronjob_scripts/.python-deps-updated.timestamp ## Publish a new release
 	(cd cargo-quickinstall/ && cargo release patch --execute --no-push)
 	git push origin HEAD:release --tags
-	make recheck
+	$(MAKE) recheck
+
+cronjob_scripts/requirements.txt: cronjob_scripts/pyproject.toml ## Compile the python dependencies from pyproject.toml into requirements.txt
+	(cd cronjob_scripts && uv pip compile pyproject.toml --python-version=3.8 --output-file requirements.txt)
+
+# install python dependencies and then record that we've done so so we don't do it again
+# WARNING: this will mess with whatever python venv you happen to be in.
+# this is run on the github actions runner so we can't use uv
+cronjob_scripts/.python-deps-updated.timestamp: cronjob_scripts/requirements.txt
+	python --version
+	pip install -r cronjob_scripts/requirements.txt
+	touch cronjob_scripts/.python-deps-updated.timestamp
 
 .PHONY: windows
-windows: ## trigger a windows build
-	RECHECK=1 TARGET_ARCH=x86_64-pc-windows-msvc ./trigger-package-build.sh
+windows: cronjob_scripts/.python-deps-updated.timestamp ## trigger a windows build
+	RECHECK=1 TARGET_ARCH=x86_64-pc-windows-msvc python cronjob_scripts/trigger-package-build.py
 
 .PHONY: mac
-mac: ## trigger a mac build
-	RECHECK=1 TARGET_ARCH=x86_64-apple-darwin ./trigger-package-build.sh
+mac: cronjob_scripts/.python-deps-updated.timestamp ## trigger a mac build
+	RECHECK=1 TARGET_ARCH=x86_64-apple-darwin python cronjob_scripts/trigger-package-build.py
 
 .PHONY: m1
-m1: ## trigger a mac m1 build
-	RECHECK=1 TARGET_ARCH=aarch64-apple-darwin ./trigger-package-build.sh
+m1: cronjob_scripts/.python-deps-updated.timestamp ## trigger a mac m1 build
+	RECHECK=1 TARGET_ARCH=aarch64-apple-darwin python cronjob_scripts/trigger-package-build.py
 
 .PHONY: linux
-linux: ## trigger a linux build
-	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-gnu ./trigger-package-build.sh
+linux: cronjob_scripts/.python-deps-updated.timestamp ## trigger a linux build
+	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-gnu python cronjob_scripts/trigger-package-build.py
 
 .PHONY: linux-musl
-linux-musl: ## trigger a musl libc-based linux build
-	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-musl ./trigger-package-build.sh
-
-.PHONY: exclude
-exclude: ## recompute excludes, but don't push anywhere (see /tmp/cargo-quickinstall-* for repos)
-	REEXCLUDE=1 ./trigger-package-build.sh
+linux-musl: cronjob_scripts/.python-deps-updated.timestamp ## trigger a musl libc-based linux build
+	RECHECK=1 TARGET_ARCH=x86_64-unknown-linux-musl python cronjob_scripts/trigger-package-build.py
 
 .PHONY: recheck
-recheck: ## recompute excludes and start from the top
-	RECHECK=1 ./trigger-package-build.sh
+recheck: cronjob_scripts/.python-deps-updated.timestamp ## build ourself and some random packages on all arches
+	RECHECK=1 TARGET_ARCH=all python cronjob_scripts/trigger-package-build.py
+
+.PHONY: trigger-all
+trigger-all: cronjob_scripts/.python-deps-updated.timestamp ## build some random packages on all arches
+	TARGET_ARCH=all python cronjob_scripts/trigger-package-build.py
+
+.PHONY: test-cronjob-scripts
+test-cronjob-scripts: cronjob_scripts/.python-deps-updated.timestamp ## run the tests for the python cronjob_scripts
+	python -m unittest discover -s cronjob_scripts
 
 .PHONY: help
 help: ## Display this help screen

README.md

@@ -45,7 +45,7 @@ Edit the command however you need, and paste it into your CI pipeline.
 
 ## Supported targets
 
-Check [supported-targets](/supported-targets) for lists of targets quickinstall
+Check [supported-targets](/cronjob_scripts/trigger-package-build.py) for lists of targets quickinstall
 can build for.
 
 ## Limitations

cronjob_scripts/.python-version

@@ -0,0 +1 @@
+3.8

cronjob_scripts/README.md

@@ -0,0 +1,7 @@
+# Scripts
+
+This folder contains python scripts for use in cargo-quickinstall github actions for triggering builds.
+
+This code has some quite heavy python dependencies and strong assumptions about running on unix, so we should not use it on the github actions runners that actually do the package building.
+
+TODO: make a build_scripts/ folder and move all of the package building scripts into there (converting to python as desired).

cronjob_scripts/architectures.py

@@ -0,0 +1,51 @@
+import os
+import subprocess
+
+
+TARGET_ARCH_TO_BUILD_OS = {
+    "x86_64-apple-darwin": "macos-latest",
+    "aarch64-apple-darwin": "macos-latest",
+    "x86_64-unknown-linux-gnu": "ubuntu-20.04",
+    "x86_64-unknown-linux-musl": "ubuntu-20.04",
+    "x86_64-pc-windows-msvc": "windows-latest",
+    "aarch64-pc-windows-msvc": "windows-latest",
+    "aarch64-unknown-linux-gnu": "ubuntu-20.04",
+    "aarch64-unknown-linux-musl": "ubuntu-20.04",
+    "armv7-unknown-linux-musleabihf": "ubuntu-20.04",
+    "armv7-unknown-linux-gnueabihf": "ubuntu-20.04",
+}
+
+
+def get_build_os(target_arch: str) -> str:
+    try:
+        return TARGET_ARCH_TO_BUILD_OS[target_arch]
+    except KeyError:
+        raise ValueError(f"Unrecognised target arch: {target_arch}")
+
+
+def get_target_architectures() -> list[str]:
+    target_arch = os.environ.get("TARGET_ARCH", None)
+    if target_arch in TARGET_ARCH_TO_BUILD_OS:
+        return [target_arch]
+
+    if target_arch == "all":
+        return list(TARGET_ARCH_TO_BUILD_OS.keys())
+
+    rustc_version_output = subprocess.run(
+        ["rustc", "--version", "--verbose"], capture_output=True, text=True
+    )
+
+    assert (
+        rustc_version_output.returncode == 0
+    ), f"rustc --version --verbose failed: {rustc_version_output}"
+
+    host_values = [
+        line.removeprefix("host: ")
+        for line in rustc_version_output.stdout.splitlines()
+        if line.startswith("host: ")
+    ]
+    assert (
+        len(host_values) == 1
+    ), f"rustc did not tell us its host, or told us multiple: {rustc_version_output}"
+
+    return host_values

cronjob_scripts/checkout_worktree.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import subprocess
+
+
+def checkout_worktree_for_arch(target_arch: str):
+    """
+    Checkout a git worktree for the given target_arch, in /tmp.
+
+    This is required for reading the exclude files.
+
+    This is lifted directly from the old trigger-package-build.sh script, and is only expected to
+    work on linux/macos with dash/bash.
+    """
+    worktree_path = f"/tmp/cargo-quickinstall-{target_arch}"
+    bash_script = f"""
+        set -eux
+
+        rm -rf {worktree_path}
+        git worktree remove -f {worktree_path} || true
+        git branch -D "trigger/{target_arch}" || true
+
+        git worktree add --force --force {worktree_path}
+        cd {worktree_path}
+
+        if git fetch origin "trigger/{target_arch}"; then
+            git checkout "origin/trigger/{target_arch}" -B "trigger/{target_arch}"
+        elif ! git checkout "trigger/{target_arch}"; then
+            # New branch with no history. Credit: https://stackoverflow.com/a/13969482
+            git checkout --orphan "trigger/{target_arch}"
+            git rm --cached -r . || true
+            git commit -m "Initial Commit" --allow-empty
+            git push origin "trigger/{target_arch}"
+        fi
+    """
+    subprocess.run(bash_script, shell=True, check=True, text=True)
+    return worktree_path
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <target_arch>")
+        sys.exit(1)
+
+    worktree_path = checkout_worktree_for_arch(sys.argv[1])
+    print(f"checked out to {worktree_path}")

cronjob_scripts/get_latest_version.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+import functools
+from typing import TypedDict
+import requests
+
+import semver
+
+
+class CrateVersionDict(TypedDict):
+    """
+    A returned row from the crates.io index API.
+
+    Note that the returned row also includes all of the fields from the jsonschema at
+    https://doc.rust-lang.org/cargo/reference/registry-index.html#json-schema
+    but I can't be bothered typing them right now.
+    """
+
+    name: str
+    vers: str
+    features: dict[str, list[str]]
+
+
+def get_index_url(crate: str):
+    """
+    Packages with 1 character names are placed in a directory named 1.
+    Packages with 2 character names are placed in a directory named 2.
+    Packages with 3 character names are placed in the directory 3/{first-character} where {first-character} is the first character of the package name.
+    All other packages are stored in directories named {first-two}/{second-two} where the top directory is the first two characters of the package name, and the next subdirectory is the third and fourth characters of the package name. For example, cargo would be stored in a file named ca/rg/cargo.
+    -- https://doc.rust-lang.org/cargo/reference/registry-index.html#index-files
+    """
+    if len(crate) == 0:
+        raise ValueError("Empty crate name")
+    if len(crate) == 1:
+        return f"https://index.crates.io/1/{crate}"
+    elif len(crate) == 2:
+        return f"https://index.crates.io/2/{crate}"
+    elif len(crate) == 3:
+        return f"https://index.crates.io/3/{crate[0]}/{crate}"
+    else:
+        return f"https://index.crates.io/{crate[:2]}/{crate[2:4]}/{crate}"
+
+
+@functools.lru_cache
+def get_latest_version(crate: str) -> CrateVersionDict | None:
+    """
+    Calls the crates.io index API to get the latest version of the given crate.
+
+    There is no rate limit on this api, so we can call it as much as we like.
+    """
+    url = get_index_url(crate)
+
+    response = requests.get(url)
+    if response.status_code == 404:
+        print(f"No crate named {crate}")
+        return None
+    response.raise_for_status()
+
+    max_version: CrateVersionDict | None = None
+    max_parsed_version: semver.VersionInfo | None = None
+    for line in response.text.splitlines():
+        version = json.loads(line)
+        parsed_version = semver.VersionInfo.parse(version["vers"])
+        if version["yanked"] or parsed_version.prerelease:
+            continue
+
+        if max_parsed_version is None or parsed_version > max_parsed_version:
+            max_version = version
+            max_parsed_version = parsed_version
+
+    return max_version
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <crate>")
+        sys.exit(1)
+
+    version = get_latest_version(sys.argv[1])
+    if version is not None:
+        print(version["vers"])

cronjob_scripts/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "cronjob-scripts"
+version = "0.0.0"
+description = "Scripts for use in cargo-quickinstall github actions"
+readme = "README.md"
+# This is what ubuntu-latest has installed by default
+requires-python = ">=3.8"
+dependencies = [
+    "influxdb3-python>=0.8.0",
+    "requests>=2.32.3",
+    "semver>=3.0.2",
+]