diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..04f2a2e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,48 @@ +name: Build cache + +on: + workflow_dispatch: {} + push: + branches: + - main + paths: + - ubuntu-22.04/** + schedule: + - cron: "0 0 * * *" + +jobs: + build: + runs-on: ubuntu-22.04 + permissions: + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup spack + uses: spack/setup-spack@v2.1.1 + with: + ref: develop + + - name: Add mochi-spack-packages + run: | + git clone https://github.com/mochi-hpc/mochi-spack-packages + spack -e ubuntu-22.04 repo add mochi-spack-packages + + - name: Install spack environment + run: | + spack -e ubuntu-22.04 install + + - name: Show spack-installed packages + run: | + spack -e ubuntu-22.04 find -dlv + + - name: Push packages to buildcache and update index + if: ${{ !cancelled() }} + run: | + spack -e ubuntu-22.04 mirror set --push \ + --oci-username ${{ github.actor }} \ + --oci-password "${{ secrets.GITHUB_TOKEN }}" mochi-buildcache + spack -e ubuntu-22.04 buildcache push --base-image ubuntu:22.04 \ + --unsigned --update-index mochi-buildcache diff --git a/README.md b/README.md new file mode 100644 index 0000000..ac9e022 --- /dev/null +++ b/README.md @@ -0,0 +1,179 @@ +# Mochi Spack Buildcache + +This repository is a mirror of pre-built Mochi spack packages. + +## How to use + +To use this mirror, you may use the following command with spack to +make the mirror available globally to all your spack environments. + +``` +$ spack mirror add --unsigned mochi oci://ghcr.io/mochi-hpc/mochi-spack-buildcache +``` + +Alternatively, you can also add the following in a spack.yaml +file for an environment. This will make the mirror visible to that +environment only. + +``` +spack: + ... + mirror: + mochi: + url: oci://ghcr.io/mochi-hpc/mochi-spack-buildcache + signed: false +``` + +## Contributing + +### From another Mochi repository + +To allow another repository inside the [mochi-hpc](https://github.com/mochi-hpc) +organization to push specs to the buildcache from a github action, +follow the steps hereafter. + +First, navigate to the buildcache's +[settings](https://github.com/orgs/mochi-hpc/packages/container/mochi-spack-buildcache/settings) +and on the right of Manage Actions access, click the Add Repository button. Select the +repository to add, and once it appears on the list bellow, change its role to "Write". + +Then, in your project's github repository, write a `spack.yaml` file to be used +for building your project's dependencies. Here is an example to start from: + +``` +spack: + specs: + - mochi-margo + - mercury~checksum~boostsys ^libfabric fabrics=tcp,rxm + concretizer: + unify: true + reuse: true + modules: + prefix_inspections: + lib: [LD_LIBRARY_PATH] + lib64: [LD_LIBRARY_PATH] + mirrors: + mochi: + url: oci://ghcr.io/mochi-hpc/mochi-spack-buildcache + signed: false + config: + install_tree: + padded_length: 128 +``` + +Make sure the mochi mirror is listed and `config:install_tree:padded_length` is set to 128. +Add any specs you may need. This environment file will allow github actions that use it to +rely on the buildcache to pull dependencies. + +Finally, here is an example of a github action that installs an environment +from the spack.yaml file (which is assumed to live in a `tests` folder, but +you can change that in via the `ENVNAME` environment variable at the beginning +of the file) and pushes its installed specs to the cache. + +``` +name: Build + +env: + ENVNAME: tests + +on: + workflow_dispatch: {} + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-22.04 + permissions: + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup spack + uses: spack/setup-spack@v2.1.1 + with: + ref: develop + color: true + path: spack + + - name: Add mochi-spack-packages + run: | + git clone https://github.com/mochi-hpc/mochi-spack-packages + spack -e $ENVNAME repo add mochi-spack-packages + + - name: Install spack environment + run: | + spack -e $ENVNAME install + + - name: Show spack-installed packages for debugging + run: | + spack -e $ENVNAME find -dlv + + - name: Extra steps + run: | + echo "Put more steps here such as building/testing your code" + + - name: Push packages to buildcache and update index + if: ${{ !cancelled() }} + run: | + spack -e $ENVNAME mirror set --push \ + --oci-username ${{ github.actor }} \ + --oci-password "${{ secrets.GITHUB_TOKEN }}" mochi + spack -e $ENVNAME buildcache push --base-image ubuntu:22.04 \ + --unsigned --update-index mochi +``` + +### Manually + +You may sometimes want to manually push some specs to the cache, for +instance if these specs have been build on a supercomputer and you want +to cache them. To do so, follow the steps hereafter. + +First, you need to be a member of the +[mochi-buildcache-maintainers](https://github.com/orgs/mochi-hpc/teams/mochi-buildcache-maintainers) +team. If you are not, request to be added by posting an issue on this repository. + +Second, in your [Developer Settings](https://github.com/settings/apps), +click "Personal access tokens" on the left, then "Tokens (classic)". +Click "Generate new token" > "Generate new token (classic)". +Call your token `MOCHI_BUILDCACHE_TOKEN`, set an appropriate expiration, +then select `write:packages` as scope. Click the "Generate token" button at +the bottom of the page, then copy your token. + +Warning: tokens are like passwords that, combined with your username, +will allow anyone who has it to perform the operations it was given the scope for. +Hence, make sure not to publish your token anywhere public and if you do by mistake, +revoke the token immediately by going back to the above page and deleting it. + +Now that you have your token, create an environment from a `spack.yaml` file +(see above for an example, make sure you have the `mirrors` entry and the + `config:install_tree:padded_length` set to 128). Activate your environment +and install the specs that it defines. + +Still with the environment activated, you can now add the token to your mirror +setup as follows (replacing `` and `` with your github username +and your generated token). + +``` +$ spack mirror set --push --oci-username --oci-password mochi +``` + +This will add the token (in clear) in a file in the currently activated +environment, hence remember the warning above and make sure that your spack +installation (or wherever your environment lives) is correctly protected. + +You can now push push specs into the buildcache. + +``` +$ spack buildcache push --update-index --unsigned mochi +``` + +Ommitting the specs will push all the specs from the environment. + +For a more details about the way buildcache work, please refer to +the [spack documentation](https://spack.readthedocs.io/en/latest/binary_caches.html). diff --git a/ubuntu-22.04/spack.yaml b/ubuntu-22.04/spack.yaml new file mode 100644 index 0000000..e81545f --- /dev/null +++ b/ubuntu-22.04/spack.yaml @@ -0,0 +1,39 @@ +# This file defines the spack environment used by github actions +# (tests, codeql, and codecov) when building and testing the code. +# In particular the "config" field sets the installation path and +# spack caches to ~/.spack-ci so that they can be retrieved using +# the github cache action across runs. +spack: + specs: + - mochi-margo + - mochi-thallium + - mochi-abt-io + - mochi-ch-placement + - mercury~checksum~boostsys + - libfabric fabrics=tcp,rxm + - mochi-bedrock+abtio+ssg+mona+mpi + - mochi-mona + - mochi-poesie+lua+python~js+bedrock + - mochi-remi+bedrock + - mochi-ssg+mpi + - mochi-warabi+bedrock+remi + - mochi-yokan+bedrock+remi+berkeleydb+leveldb+lmdb+rocksdb+tkrzw+gdbm+unqlite+lua+python + - py-mochi-margo + concretizer: + unify: when_possible + reuse: dependencies + modules: + prefix_inspections: + lib: [LD_LIBRARY_PATH] + lib64: [LD_LIBRARY_PATH] + mirrors: + mochi-buildcache: + url: oci://ghcr.io/mochi-hpc/mochi-spack-buildcache + signed: false + config: + install_tree: + padded_length: 128 + packages: + all: + providers: + mpi: [ mpich ]