diff --git a/update_conda_lock/action.yml b/update_conda_lock/action.yml new file mode 100644 index 0000000..2119fdd --- /dev/null +++ b/update_conda_lock/action.yml @@ -0,0 +1,129 @@ +name: 'Conda Lock Updating Action' +description: 'Action Pushes a Branch with Conda Lock Update' + +inputs: + commit_message: + description: 'Message of the bump commit' + required: true + default: '[BOT] Bump Conda Lock' + user_name: + description: 'Name of the user creating commit' + required: true + default: 'CONDA_LOCK_UPDATING_BOT' + user_email: + description: 'Email address of the user creating commit' + required: true + default: '<>' + branch_name: + description: "Name of the pushed branch; it will be suffixed with GH Action's Run ID" + required: true + default: 'update_lock' + environment_file: + description: 'Path of the `environment.yml` file' + required: true + default: 'environment.yml' + conda_lock_file: + description: 'Path of the Conda Lock file (needs to have txt/yml/yaml extension)' + required: true + default: 'conda_lock.yml' + pr_title: + description: 'Title of the Pull Request; it will be suffixed with UTC time and date' + required: true + default: '[BOT] Bump Conda Lock' + github_token: + description: 'GitHub Access Token; use Private Access Token to trigger workflows issuing the Pull Request' + required: true + +runs: + using: "composite" + steps: + - id: lock-update + shell: bash + run: | + echo "::group::Check git" + which git + git --version + echo "::endgroup::" + + echo "::group::Check conda" + which conda + conda -V + echo + echo "Packages in the current environment:" + conda list + echo "::endgroup::" + + echo "::group::Check python" + which python3 + python3 -V + echo + echo "Python modules installed in pip:" + python3 -m pip list + echo "::endgroup::" + + echo "::group::Check curl" + which curl + curl -V + echo "::endgroup::" + + echo "::group::Capture Base Branch Name" + set -x + export PR_BASE=$(git branch --show-current) + if [ "$PR_BASE" = "" ]; then + echo "Capturing base branch failed!" + exit 1 + fi + set +x + echo "::endgroup::" + + echo "::group::Create New Branch" + export BRANCH_NAME="${{ inputs.branch_name }}_${{ github.run_id }}" + git checkout -b "$BRANCH_NAME" + echo "::endgroup::" + + echo "::group::Install ruamel.yaml" + python3 -m pip install ruamel.yaml + echo "::endgroup::" + + echo "::group::Update Conda Lock" + set -x + # Export variables used by the script + export BOT_CONDA_LOCK="${{ inputs.conda_lock_file }}" + export BOT_ENV_YML="${{ inputs.environment_file }}" + + EXIT_CODE=0 + python3 $GITHUB_ACTION_PATH/update_lock.py || EXIT_CODE=$? + echo "Script returned code: $EXIT_CODE" + if [ $EXIT_CODE -eq 3 ]; then + exit 0 + elif [ $EXIT_CODE -ne 0 ]; then + exit $EXIT_CODE + fi + set +x + echo "::endgroup::" + + echo "::group::Add And Commit Changes" + set -x + git config user.name "${{ inputs.user_name }}" + git config user.email "${{ inputs.user_email }}" + git add "$BOT_CONDA_LOCK" + git commit -m "${{ inputs.commit_message }}" + set +x + echo "::endgroup::" + + echo "::group::Push Changes" + git push -u origin "$BRANCH_NAME" + echo "::endgroup::" + + echo "::group::Create a Pull Request" + # Export variables used by the script + export GITHUB_TOKEN="${{ inputs.github_token }}" + set -x + export GITHUB_REPOSITORY="${{ github.repository }}" + # PR_BASE has been already exported before the `git checkout -b` command + export PR_HEAD="$BRANCH_NAME" + export PR_TITLE="${{ inputs.pr_title }} $(date -u +'%Y-%m-%d %H:%M %p %Z')" + + python3 $GITHUB_ACTION_PATH/create_pull_request.py + set +x + echo "::endgroup::" diff --git a/update_conda_lock/create_pull_request.py b/update_conda_lock/create_pull_request.py new file mode 100644 index 0000000..6845548 --- /dev/null +++ b/update_conda_lock/create_pull_request.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import json +import requests +import sys +from os import environ + + +def create_pull_request(gh_repo, gh_token, pr_base, pr_head, pr_title): + response = requests.post( + 'https://api.github.com/repos/' + gh_repo + '/pulls', + headers={ + 'Authorization': 'token ' + gh_token, + 'Accept': 'application/vnd.github.v3+json', + }, + json={ + 'base': pr_base, + 'head': pr_head, + 'title': pr_title, + }, + ) + if response.status_code != 201: + print('ERROR: Pull Request creation failed with status: ' + + str(response.status_code) + ' ' + response.reason) + print() + print('GitHub API response data was:') + json.dump(response.json(), sys.stdout, indent=2) + print() + sys.exit(1) + else: + print('Pull Request created successfully!') + print('It\'s available at: ' + response.json()['html_url']) + + +if __name__ == '__main__': + create_pull_request( + environ['GITHUB_REPOSITORY'], + environ['GITHUB_TOKEN'], + environ['PR_BASE'], + environ['PR_HEAD'], + environ['PR_TITLE'], + ) diff --git a/update_conda_lock/update_lock.py b/update_conda_lock/update_lock.py new file mode 100644 index 0000000..d4a7747 --- /dev/null +++ b/update_conda_lock/update_lock.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +from io import StringIO +from os import environ +from os.path import dirname, exists, isdir, join, splitext +from subprocess import run, DEVNULL, PIPE, CalledProcessError +from sys import exit +from re import match, search, sub +from tempfile import NamedTemporaryFile + +from ruamel.yaml import YAML +yaml = YAML() +yaml.allow_duplicate_keys = True + +def _run(cmd_string, multiword_last_arg='', return_stdout=False, **kwargs): + cmd = cmd_string.split() + ([multiword_last_arg] + if multiword_last_arg else []) + if return_stdout: + try: + return run(cmd, check=True, encoding='utf-8', stdout=PIPE, + **kwargs).stdout + except CalledProcessError as e: + print(e.output) + raise + else: + return run(cmd, check=True, **kwargs) + +def _get_env(env_name): + if env_name not in environ: + print('ERROR: Required environment variable not found: ' + env_name + '!') + return None + env_var = environ[env_name] + print('* ' + env_name + ': ' + env_var) + return env_var + +# Returns True if lock file has been updated, False otherwise +def try_updating_lock_file(path, lock_yml): + print('Trying to update `' + path + '`...') + with StringIO() as tmp_stream: + yaml.dump(lock_yml, tmp_stream) + new_lock = tmp_stream.getvalue() + try: + with open(path, 'r') as f: + old_lock = f.read() + if old_lock == new_lock: + print(path + ' is up to date.') + print() + return False + except FileNotFoundError: + print(path + ' doesn\'t exist; it will be created.') + with open(path, 'w') as f: + f.write(new_lock) + print(path + ' has been updated successfully!') + print() + return True + +def analyze_pip_requirement(requirement, analyzed_file_dir): + path_match = match(r'-r (file:)?(.*)', requirement) + if path_match is None: + return [ requirement ] + else: # `requirement` includes some additional `requirements.txt` file + # `-r PATH` is relative to the environment file + req_path = join(analyzed_file_dir, path_match.group(2)) + print('Found additional pip requirements file: ' + req_path) + with open(req_path, 'r') as f: + file_requirements = [] + for req_line in f.readlines(): + file_requirements.extend( + analyze_pip_requirement(req_line, dirname(req_path)) + ) + return file_requirements + +def get_all_pip_dependencies(pip_dependencies, analyzed_file_dir): + all_pip_dependencies = [] + for pip_dependency in pip_dependencies: + all_pip_dependencies.extend( + analyze_pip_requirement(pip_dependency, analyzed_file_dir) + ) + return all_pip_dependencies + +def extract_pip_dependencies(env_yml_path): + with open(env_yml_path, 'r') as f: + env_yml = yaml.load(f.read()) + + pip_dependencies = None + for dependency in env_yml['dependencies']: + # `- pip:` line becomes a dict-like object with `pip` key after parsing + if isinstance(dependency, dict) and 'pip' in dependency.keys(): + # `pip:` key is replaced with `pip` package to have it installed + # even when there was only `pip:` key in `environment.yml` + env_yml['dependencies'].remove(dependency) + env_yml['dependencies'].append('pip') + env_yml_pip_dependencies = list(dependency['pip']) + + # Save `environment.yml` without pip requirements + env_yml_path = 'bot-env.yml' + with open(env_yml_path, 'w') as f: + yaml.dump(env_yml, f) + return (env_yml['name'], env_yml_path, env_yml_pip_dependencies) + + +def get_local_pip_dependencies(pip_dependencies, root_dir): + local_pip_dependencies = [] + local_pip_deps_names = [] + for dependency in pip_dependencies: + dependency = dependency.strip() + # Handle comments + if dependency.startswith('#'): + continue + only_dependency = sub(r'(.*)\s+#.*$', r'\1', dependency) + + # Find core of the dependency line without version etc. + core_match = search(r'(^|\s)([^\s-][^\s=<>~!;]+)', only_dependency) + if core_match is not None: + dependency_path = join(root_dir, core_match.group(2)) + if isdir(dependency_path): + setup_path = join(dependency_path, 'setup.py') + if exists(setup_path): + dependency_name = _run('python setup.py --name', + cwd=dependency_path, return_stdout=True).strip() + local_pip_deps_names.append(dependency_name) + local_pip_dependencies.append(dependency) + return (local_pip_dependencies, local_pip_deps_names) + + +class CondaEnvironmentContext: + def __init__(self, name, env_path): + self._name = name + self._env_path = env_path + + def __enter__(self): + print('Creating `' + self._name + '` environment based on `' + + self._env_path + '`...') + print() + try: + _run('conda env create -n ' + self._name + ' -f ' + self._env_path) + except: + print('ERROR: Creating `' + self._name + '` environment failed!') + print('Please remove any environment with such name, if exists.') + print() + exit(1) + + def __exit__(self, exc_type, exc_value, traceback): + print('Removing `' + self._name + '` Conda environment... ', end='') + _run('conda env remove -n ' + self._name, stdout=DEVNULL, + stderr=DEVNULL) + print('done!') + print() + + +def main(): + print('Environment variables used are:') + env_yml_path = _get_env('BOT_ENV_YML') + conda_lock_path = _get_env('BOT_CONDA_LOCK') + print() + if None in [env_yml_path, conda_lock_path]: + exit(1) + + # Conda only supports creating environments from .txt/.yml/.yaml files + _, conda_lock_ext = splitext(conda_lock_path) + if conda_lock_ext not in ['.txt', '.yml', '.yaml']: + print('ERROR: Invalid conda lock extension (`' + conda_lock_ext + + '`); it must be `.txt`, `.yml` or `.yaml`!') + exit(1) + + (conda_env, pipless_env_yml_path, env_yml_pip_deps) = extract_pip_dependencies( + env_yml_path) + + with CondaEnvironmentContext(conda_env, pipless_env_yml_path): + conda_lock = _run('conda run -n ' + conda_env + ' conda env export', + return_stdout=True) + conda_lock_yaml = yaml.load(conda_lock) + print('Conda packages captured.') + print() + + # Lock pip dependencies + if env_yml_pip_deps: + pip_cmd = 'conda run --no-capture-output -n ' + conda_env + ' python -I -m pip ' + all_pip_deps = get_all_pip_dependencies(env_yml_pip_deps, + dirname(env_yml_path)) + + # Local pip dependencies will be uninstalled and copied in the original + # form as freezing breaks them (git handles their versioning after all). + (local_deps, local_deps_names) = get_local_pip_dependencies( + all_pip_deps, dirname(env_yml_path)) + + print('Installing pip dependencies...') + print() + with NamedTemporaryFile('w+', encoding='utf-8', newline='\n') as f: + f.write('\n'.join(env_yml_pip_deps)) + f.flush() + + # Possible requirements file paths are relative to `environment.yml` + env_yml_dir = dirname(env_yml_path) + _run(pip_cmd + 'install -r ' + f.name, cwd=env_yml_dir or '.') + print() + + # Uninstall local packages + if local_deps and local_deps_names: + print('Uninstalling local pip packages (they were installed ' + + 'only to lock their dependencies\' versions)...') + print() + for local_pkg in local_deps_names: + _run(pip_cmd + 'uninstall --yes ' + local_pkg) + print() + + pip_locked_pkgs = [] + for pip_spec in _run(pip_cmd + 'freeze', return_stdout=True).splitlines(): + if pip_spec: + # Ignore pip packages installed by Conda + # (lines: 'NAME @ file://PATH/work') + conda_pkg_match = match(r'(\S+) @ file://.*/work.*', pip_spec) + if conda_pkg_match is not None: + print('Ignoring pip package installed by Conda: ' + + conda_pkg_match.group(1)) + continue + pip_locked_pkgs.append(pip_spec) + + # Add local packages + if local_deps: + pip_locked_pkgs.extend(local_deps) + + # Add locked pip packages to the `conda env export` yaml output + if pip_locked_pkgs: + conda_lock_yaml['dependencies'].append({'pip': pip_locked_pkgs}) + + print() + print('Pip packages captured.') + print() + + # Apply yaml offset used by `conda env export` + yaml.indent(offset=2) + if not try_updating_lock_file(conda_lock_path, conda_lock_yaml): + exit(3) + +if __name__ == '__main__': + main()