From 818fb91ab90919bdc6f95bb7af4a66b1b4ddec41 Mon Sep 17 00:00:00 2001 From: Dawa Ometto Date: Thu, 2 May 2024 16:29:03 +0200 Subject: [PATCH] Add support for latest system python to pyenv and python-workbench (#122) * role pip: minor refactor * role pyenv: support system-latest * python-workbench: support system-latest * Update pyenv and python-workbench docs --- docs/playbooks/python-workbench.md | 13 ++++--- docs/roles/pyenv.md | 10 ++++-- playbooks/python-workbench.yml | 2 +- playbooks/roles/pip/tasks/main.yml | 1 + playbooks/roles/pyenv/defaults/main.yml | 7 ++++ .../molecule/pyenv-system_python/converge.yml | 10 ++++++ .../molecule/pyenv-system_python/molecule.yml | 10 ++++++ .../molecule/pyenv-system_python/verify.yml | 34 +++++++++++++++++++ playbooks/roles/pyenv/tasks/main.yml | 28 +++++++++++++++ .../roles/pyenv/templates/pyenv-install.sh.j2 | 12 +++++-- 10 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 playbooks/roles/pyenv/molecule/pyenv-system_python/converge.yml create mode 100644 playbooks/roles/pyenv/molecule/pyenv-system_python/molecule.yml create mode 100644 playbooks/roles/pyenv/molecule/pyenv-system_python/verify.yml diff --git a/docs/playbooks/python-workbench.md b/docs/playbooks/python-workbench.md index 148bc5ec..4d216ff8 100644 --- a/docs/playbooks/python-workbench.md +++ b/docs/playbooks/python-workbench.md @@ -7,24 +7,29 @@ Installs a Python development environment equiped with to allow the user to select an arbitrary Python version and *[poetry](https://pypi.org/project/poetry/)* to facilitate Python module dependency management. +*[miniconda](https://docs.anaconda.com/free/miniconda/index.html)* +to facilitate general dependency management. ## Requires -Linux operating system +Ubuntu . ## Description -The Linux distribution influences which Python3 and pip3 versions are installed system-wide. This component therefore installs `pyenv` and `poetry` via the `runonce` role, so that each user on the system can manage their own Python environment. `pyenv` and `poetry` are installed (in userspace) the first time the user logs in. +The Linux distribution influences which Python3 and pip3 versions are installed system-wide. Users should not install python packages using the system python interpreter, since this may break system packages. This component therefore installs `pyenv` and via the `runonce` role, so that each user on the system can manage their own Python environment. `pyenv` and `poetry` are installed (in userspace) the first time the user logs in. + +For additional development convenience, `poetry` and `miniconda` are also intalled on a per-user basis. If users want to use `miniconda`, they have to manually run `conda init` in their shell (this is not done by default, because it may interfere with users' workflows if they do not want to use `conda`). ## Variables -- `default_python_version`: String. The version of Python to be automatically installed for each in user via `pyenv`, at first login. This version is also set as the default version for that user (with `pyenv global`). If this parameter is omitted or set to `system`, `pyenv` will not install a specific version but install use the system's python version by default . Default: `3.9.18`. +- `default_python_version`: String. The version of Python to be automatically installed for each in user via `pyenv`, at first login. This version is also set as the default version for that user (with `pyenv global`). If this parameter is omitted or set to `system`, `pyenv` will not install a specific version but install use the system's python version by default . If this parameter is set to `system-latest`, the latest version of Python available from `apt` will be fetched and set as `pyenv`'s global version for each user. Default: `system-latest`. ## See also - role [runonce](../roles/runonce.md) - role [pyenv](../roles/pyenv.md) - role [runonce](../roles/poetry.md) +- role [miniconda](../roles/miniconda.md) ## History -2021 Written by Ton Smeele (Utrecht University) +2021-2024 Written by Ton Smeele and Dawa Ometto (Utrecht University) [back to index](../index.md#Playbooks) diff --git a/docs/roles/pyenv.md b/docs/roles/pyenv.md index b6ce69c2..8cde4ace 100644 --- a/docs/roles/pyenv.md +++ b/docs/roles/pyenv.md @@ -11,15 +11,21 @@ Pyenv is a Python version management tool, which lets you change the global Pyth ## Description This role will install pyenv for each user individually. For both yum and apt based package managers, all dependencies (primarily needed for compiling python versions) are first downloaded. The installation is then done through the pyenv installer script which is run once when the user logs in. +The role supports installing the latest version of python available in the OS's package manager and registering that python version with `pyenv`. This is useful because OS distributions generally ship with older python versions by default, and when installing an addtional specific python version (e.g. `apt install python3.12` ), `pyenv` will not know about it by default. See the special `system-latest` value for the `default_python_version` variable below. **Note**: this is only supported on Ubuntu at the moment. + ## Variables -- `default_python_version`: String. The version of Python to be automatically installed for each in user at first login. This version is also set as the default version for that user (with `pyenv global`). Default: `system` (don't install a new version, but use the system's python version by default). +- `default_python_version`: String. The version of Python to be automatically installed for each in user at first login. This version is also set as the default version for that user (with `pyenv global`). Default: `system` (don't install a new version, but use the system's python version by default). Possible values: + + * `3.11` (installs latest `3.11` python, e.g. `3.11.1`) + * `3.8.4.1` (install specific version). + * Special value: `system-latest`. Will use the OS's package manager (e.g. `apt`) to fetch the latest packaged python version, and this version will be used as `pyenv`'s global python version instead of the older, default system python version. ## See also - [runeonce role](../roles/runonce.md) ## History -2022 Written by Sytse Groenwold (Utrecht University) +2022-2024 Written by Sytse Groenwold and Dawa Ometto (Utrecht University) [back to index](../index.md#Roles) diff --git a/playbooks/python-workbench.yml b/playbooks/python-workbench.yml index d42c2950..c88498c9 100644 --- a/playbooks/python-workbench.yml +++ b/playbooks/python-workbench.yml @@ -7,5 +7,5 @@ - role: pip - role: pyenv vars: - pyenv_default_python: "{{ default_python_version | default(omit) }}" + pyenv_default_python: "{{ default_python_version | default('system-latest') }}" - role: poetry diff --git a/playbooks/roles/pip/tasks/main.yml b/playbooks/roles/pip/tasks/main.yml index 01f15ebe..2c09e7fa 100644 --- a/playbooks/roles/pip/tasks/main.yml +++ b/playbooks/roles/pip/tasks/main.yml @@ -3,3 +3,4 @@ package: name: python3-pip state: present + when: ansible_pkg_mgr == 'apt' diff --git a/playbooks/roles/pyenv/defaults/main.yml b/playbooks/roles/pyenv/defaults/main.yml index 2e7f9e7e..9fbd31c1 100644 --- a/playbooks/roles/pyenv/defaults/main.yml +++ b/playbooks/roles/pyenv/defaults/main.yml @@ -1,2 +1,9 @@ --- pyenv_default_python: "system" +pyenv_system_python_latest_versions: + Ubuntu: + '20.04': '3.9' + '22.04': '3.11' + '23.04': '3.11' + '23.10': '3.12' + '24.04': '3.12' diff --git a/playbooks/roles/pyenv/molecule/pyenv-system_python/converge.yml b/playbooks/roles/pyenv/molecule/pyenv-system_python/converge.yml new file mode 100644 index 00000000..f6d01779 --- /dev/null +++ b/playbooks/roles/pyenv/molecule/pyenv-system_python/converge.yml @@ -0,0 +1,10 @@ +--- +- name: Converge + hosts: all + gather_facts: true + tasks: + - name: Testing userspace pyenv role + include_role: + name: pyenv + vars: + pyenv_default_python: 'system-latest' diff --git a/playbooks/roles/pyenv/molecule/pyenv-system_python/molecule.yml b/playbooks/roles/pyenv/molecule/pyenv-system_python/molecule.yml new file mode 100644 index 00000000..36379814 --- /dev/null +++ b/playbooks/roles/pyenv/molecule/pyenv-system_python/molecule.yml @@ -0,0 +1,10 @@ +--- +provisioner: + name: ansible + playbooks: + converge: ./converge.yml + prepare: ../default/prepare.yml + env: + ANSIBLE_ROLES_PATH: ../../../ + ANSIBLE_PYTHON_INTERPRETER: /usr/bin/python3 +role_name_check: 1 diff --git a/playbooks/roles/pyenv/molecule/pyenv-system_python/verify.yml b/playbooks/roles/pyenv/molecule/pyenv-system_python/verify.yml new file mode 100644 index 00000000..e1e5a5c9 --- /dev/null +++ b/playbooks/roles/pyenv/molecule/pyenv-system_python/verify.yml @@ -0,0 +1,34 @@ +--- +- name: Verify + hosts: all + gather_facts: true + tasks: + - name: Get all runonce files + find: + paths: /etc/runonce.d + recurse: no + register: found_scripts + + - name: Run runonce scripts + ansible.builtin.command: su -c "{{ item.path }}" - testuser + with_items: "{{ found_scripts.files }}" + + - name: Test pyenv + ansible.builtin.shell: su -c "bash -ic 'pyenv global'" - testuser + register: pyenv_global + + - name: Test python version + ansible.builtin.shell: su -c "bash -ic 'python --version'" - testuser + register: python_version + + - name: Set expected python versions + set_fact: + expected_python_version: + '20.04': '3.9' + '22.04': '3.11' + + - name: Assert correct global python version + ansible.builtin.assert: + that: + - "pyenv_global.stdout_lines[-1] == 'system-latest'" + - "expected_python_version[ansible_distribution_version] in python_version.stdout_lines[-1]" diff --git a/playbooks/roles/pyenv/tasks/main.yml b/playbooks/roles/pyenv/tasks/main.yml index 228a6854..2742a7a4 100644 --- a/playbooks/roles/pyenv/tasks/main.yml +++ b/playbooks/roles/pyenv/tasks/main.yml @@ -37,8 +37,36 @@ - python3-openssl when: ansible_pkg_mgr == 'apt' +- name: Set latest system python fact + when: pyenv_default_python == 'system-latest' + set_fact: + _pyenv_latest_system_python: "{{ pyenv_system_python_latest_versions[ansible_distribution][ansible_distribution_version] | default('') }}" + +- name: Fail if no latest python version for this OS is defined + fail: + msg: "Python version system-latest requested, but latest python version for {{ ansible_distribution }} {{ ansible_distribution_version }} is defined." + when: pyenv_default_python == 'system-latest' and _pyenv_latest_system_python | length == 0 + +- name: Install latest system python for Ubuntu + when: pyenv_default_python == 'system-latest' and ansible_distribution == "Ubuntu" + block: + + - name: Install latest system python and corresponding venv + package: + name: "{{ item }}" + state: present + with_items: + - "python{{ _pyenv_latest_system_python }}" + - "python{{ _pyenv_latest_system_python }}-venv" + + - name: Set path to latest system python + set_fact: + _pyenv_system_python_path: "/usr/bin/python{{ _pyenv_latest_system_python }}" + - name: Add pyenv install to runonce config template: src: pyenv-install.sh.j2 dest: /etc/runonce.d/01_pyenv-install.sh mode: "0755" + vars: + system_python_path: "{{ _pyenv_system_python_path | default('') }}" diff --git a/playbooks/roles/pyenv/templates/pyenv-install.sh.j2 b/playbooks/roles/pyenv/templates/pyenv-install.sh.j2 index c136e125..95cb15f9 100644 --- a/playbooks/roles/pyenv/templates/pyenv-install.sh.j2 +++ b/playbooks/roles/pyenv/templates/pyenv-install.sh.j2 @@ -4,6 +4,7 @@ set -e # installs default python version if set DEFAULT_PYTHON_VERSION="{{ pyenv_default_python }}" +SYSTEM_PYTHON="{{ system_python_path }}" cd curl https://pyenv.run | bash @@ -23,9 +24,14 @@ INPUTEND export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" -if [[ -n "$DEFAULT_PYTHON_VERSION" && "$DEFAULT_PYTHON_VERSION" != "system" ]]; then - echo "Installing python $DEFAULT_PYTHON_VERSION from pyenv" - pyenv install "$DEFAULT_PYTHON_VERSION" +if [[ -n "$DEFAULT_PYTHON_VERSION" ]]; then + if [[ "$DEFAULT_PYTHON_VERSION" != system* ]]; then + echo "Installing python $DEFAULT_PYTHON_VERSION from pyenv" + pyenv install "$DEFAULT_PYTHON_VERSION" + fi + if [[ "$DEFAULT_PYTHON_VERSION" == "system-latest" ]]; then + "$SYSTEM_PYTHON" -m venv "$PYENV_ROOT/versions/$DEFAULT_PYTHON_VERSION" + fi pyenv global "$DEFAULT_PYTHON_VERSION" echo "Python release $DEFAULT_PYTHON_VERSION activated as 'global' pyenv release" cat >> ~/.bashrc <<'INPUTEND'