diff --git a/.github/actions/install_compatibility_layer/entrypoint.sh b/.github/actions/install_compatibility_layer/entrypoint.sh index aca33c96..b4904ddd 100755 --- a/.github/actions/install_compatibility_layer/entrypoint.sh +++ b/.github/actions/install_compatibility_layer/entrypoint.sh @@ -5,7 +5,7 @@ cat << EOF > hosts 127.0.0.1 eessi_host_arch=$(uname -m) eessi_host_os=linux EOF -ansible-playbook --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml +ansible-playbook -v --connection=local --inventory=hosts -e ansible_python_interpreter=python3 -e gentoo_prefix_path=$1 ${GITHUB_WORKSPACE}/ansible/playbooks/install.yml # A successful installation should at least have Lmod and archspec, # so let's check if we can use them. diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache new file mode 100755 index 00000000..65b4f709 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_cache @@ -0,0 +1,34 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +if [[ -f ${repository_path}/metadata/timestamp.x ]]; then + portage_current_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) +else + portage_current_timestamp=0 +fi + +ebegin "Fetching metadata timestamp for ${repository_name}" +rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/timestamp.x "${repository_path}"/metadata/timestamp.x +eend $? +portage_new_timestamp=$(cut -f 1 -d " " "${repository_path}/metadata/timestamp.x" ) + +if [[ ${portage_current_timestamp} -lt ${portage_new_timestamp} ]]; then + ebegin "Fetching pre-generated metadata cache for ${repository_name}" + rsync -aq rsync://rsync.gentoo.org/gentoo-portage/metadata/md5-cache/ "${repository_path}"/metadata/md5-cache/ + eend $? +else + einfo "Metadata cache for ${repository_name} already recent, no need to fetch it :-)" +fi + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd new file mode 100755 index 00000000..c12f6c3b --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_dtd @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +DTDDIR="${repository_path}"/metadata/dtd +ebegin "Updating DTDs" +if [[ -e ${DTDDIR} ]]; then + git -C "${DTDDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/dtd.git "${DTDDIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa new file mode 100755 index 00000000..e2a2face --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_glsa @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +GLSADIR="${repository_path}"/metadata/glsa +ebegin "Updating GLSAs" +if [[ -e ${GLSADIR} ]]; then + git -C "${GLSADIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/glsa.git "${GLSADIR}" +fi +eend "$?" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news new file mode 100755 index 00000000..92c69a63 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_news @@ -0,0 +1,17 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +NEWSDIR="${repository_path}"/metadata/news +ebegin "Updating news items" +if [[ -e ${NEWSDIR} ]]; then + git -C "${NEWSDIR}" pull -q --ff-only +else + git clone -q https://anongit.gentoo.org/git/data/gentoo-news.git "${NEWSDIR}" +fi +eend $? "Try to remove ${NEWSDIR}" diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml new file mode 100644 index 00000000..fe1ac0e9 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_gentoo_projects_xml @@ -0,0 +1,12 @@ +#!/bin/bash + +repository_name="${1}" +repository_path="${3}" + +[[ ${repository_name} == "gentoo" ]] || exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +ebegin "Updating projects.xml" +wget -q -P "${repository_path}"/metadata/ -N https://api.gentoo.org/metastructure/projects.xml +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache b/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache new file mode 100755 index 00000000..e7f5ecf0 --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/files/sync_overlay_cache @@ -0,0 +1,14 @@ +#!/bin/bash + +repository_name="${1}" + +[[ ${repository_name} == "gentoo" ]] && exit 0 + +source $(portageq envvar EPREFIX)/lib/gentoo/functions.sh + +# Number of jobs for egencache, default is number or processors. +parallel_jobs="$(nproc)" + +ebegin "Updating metadata cache for ${repository_name}" +egencache --jobs="${parallel_jobs}" --repo="${repository_name}" --update --update-use-local-desc +eend $? diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 28a81364..0454e1aa 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -58,6 +58,46 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +- name: Use git instead of rsync for the Gentoo overlay + copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/gentoo.conf" + mode: "0644" + content: | + [DEFAULT] + main-repo = gentoo + sync-git-pull-extra-opts = --quiet + + [gentoo] + priority = 1 + location = {{ gentoo_prefix_path }}/var/db/repos/gentoo + sync-uri = https://github.com/gentoo/gentoo.git + sync-type = git + auto-sync = Yes + clone-depth = 1 + +- name: Create repo.postsync.d directory + file: + path: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d" + state: directory + mode: '0755' + +- name: Add post-sync hooks for Gentoo overlay + copy: + src: "{{ item }}" + dest: "{{ gentoo_prefix_path }}/etc/portage/repo.postsync.d/{{ item }}" + mode: 0755 + with_items: + - sync_gentoo_cache + - sync_gentoo_dtd + - sync_gentoo_glsa + - sync_gentoo_news + - sync_overlay_cache + +- name: Remove the old Gentoo overlay directory + file: + state: absent + path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo" + - name: Sync the repositories portage: sync: 'yes' @@ -77,3 +117,11 @@ force: yes with_items: "{{ find_configs.results | rejectattr('files', 'equalto', []) | map(attribute='files') | list }}" + +- name: Update system set if we are running in an existing Prefix installation + portage: + changed_use: yes + deep: yes + package: "@world" + update: yes + when: startprefix.stat.exists diff --git a/test/compat_layer.py b/test/compat_layer.py index 2e2fb745..c45e35c5 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -183,6 +183,26 @@ def __init__(self): sn.assert_found(f'\n/{self.symlink_to_host}\n', self.stdout), ]) + +@rfm.simple_test +class GentooOverlayGitTest(RunInGentooPrefixTest): + def __init__(self): + # the switch to git was made in pilot version 2021.06 + self.skip_if(self.eessi_version == '2021.03') + + super().__init__() + self.descr = 'Verify that the Gentoo overlay is synced using git.' + self.command = f'emerge --info' + + gentoo_repo_dir = os.path.join(self.compat_dir, 'var', 'db', 'repos', 'gentoo') + gentoo_git_repo_info = '''gentoo + location: %s + sync-type: git + sync-uri: https://github.com/gentoo/gentoo.git''' % gentoo_repo_dir + + self.sanity_patterns = sn.assert_found(gentoo_git_repo_info, self.stdout) + + @rfm.simple_test class GlibcEnvFileTest(RunInGentooPrefixTest): def __init__(self):