From 876e8c4083a71a920702b21c3cc8007f4ced4ac7 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Tue, 18 Jun 2024 12:37:01 +0530 Subject: [PATCH] Add Meta content provider job to test opendev and github changes Meta content provider does following things: - It combines 3 content provider (operator, tcib and edpm image build) into one content provider to test all changes together. - If the changes coming from opendev, tcib and os-net-config, then it will build the DLRN packages and create gating repo. - If there is a gating and non os-net-config changes, then it will do the tcib build. - If there is a edpm-image-build change, then it will perform the image build. - All the built content will be pushed on single registry with different namespace. - The content provider will expose the same zuul return vars to dependent job to pick proper built content. - It also returns content_provider_os_registry_url var to dependent job and improve edpm_prepare role to set proper vars. - Generate gating repo only when there are repos. Note: It make changes to edpm_prepare to check edpm_image_build_output key length. If no edpm image is built by meta content provider then edpm_image_build_output will return an empty dict. Signed-off-by: Chandan Kumar --- .../meta_content_provider.yml | 150 ++++++++++++++++++ ci/playbooks/meta_content_provider/run.yml | 61 +++++++ ci/templates/projects.yaml | 1 + roles/build_openstack_packages/tasks/main.yml | 3 - .../tasks/parse_and_build_pkgs.yml | 4 + .../tasks/kustomize_and_deploy.yml | 10 ++ roles/edpm_prepare/tasks/main.yml | 4 +- scenarios/centos-9/meta_content_provider.yml | 24 +++ zuul.d/content_provider.yaml | 21 +++ zuul.d/projects.yaml | 1 + 10 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 ci/playbooks/meta_content_provider/meta_content_provider.yml create mode 100644 ci/playbooks/meta_content_provider/run.yml create mode 100644 scenarios/centos-9/meta_content_provider.yml diff --git a/ci/playbooks/meta_content_provider/meta_content_provider.yml b/ci/playbooks/meta_content_provider/meta_content_provider.yml new file mode 100644 index 0000000000..26c360b184 --- /dev/null +++ b/ci/playbooks/meta_content_provider/meta_content_provider.yml @@ -0,0 +1,150 @@ +--- +- name: Bootstrap step + ansible.builtin.import_playbook: >- + {{ + [ + ansible_user_dir, + zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir, + 'playbooks', + '01-bootstrap.yml' + ] | ansible.builtin.path_join + }} + +- name: "Run ci/playbooks/meta_content_provider/meta_content_provider.yml" + hosts: "{{ cifmw_target_host | default('localhost') }}" + gather_facts: true + tasks: + - name: Install necessary dependencies + ansible.builtin.include_role: + name: 'install_yamls_makes' + tasks_from: 'make_download_tools' + + - name: Build OpenStack Services Packages using DLRN + vars: + cifmw_bop_yum_repos_dir: "{{ cifmw_build_containers_repo_dir }}" + cifmw_bop_gating_repo_dest: "{{ cifmw_build_containers_repo_dir }}" + ansible.builtin.include_role: + name: build_openstack_packages + + - name: Check for gating repo + ansible.builtin.stat: + path: "{{ cifmw_build_containers_repo_dir }}/gating.repo" + register: _gating_repo + + - name: Deploy content provider registry + ansible.builtin.include_role: + name: registry_deploy + + - name: Construct project change list + ansible.builtin.set_fact: + zuul_change_list: "{{ zuul_change_list | default([]) + [item.project.short_name] }}" + cacheable: true + with_items: "{{ zuul['items'] }}" + when: + - zuul is defined + - "'change_url' in item" + + - name: Build openstack services container when gating repo exists + when: + - "'os-net-config' not in zuul_change_list" + - "'edpm-image-builder' not in zuul_change_list" + - _gating_repo.stat.exists + block: + - name: Build OpenStack services containers + ansible.builtin.include_role: + name: build_containers + + - name: Return registry_url + ansible.builtin.set_fact: + content_provider_os_registry_url: "{{ cifmw_build_containers_push_registry }}/{{ cifmw_build_containers_registry_namespace }}" + + - name: Build EDPM Images + when: "'edpm-image-builder' in zuul_change_list" + block: + - name: Get latest commit when no PR is provided + ansible.builtin.command: # noqa: command-instead-of-module + cmd: git show-ref --head --hash head + args: + chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/edpm-image-builder" + register: git_head_out + + - name: Set pr_sha to be used as image tag + ansible.builtin.set_fact: + pr_sha: "{{ git_head_out.stdout | trim }}" + cacheable: true + + - name: Build edpm and ipa images + ansible.builtin.include_role: + name: edpm_build_images + vars: + cifmw_edpm_build_images_tag: "{{ pr_sha }}" + + - name: Push edpm-hardened-uefi image to registry + containers.podman.podman_image: + name: "{{ item }}" + push_args: + dest: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/{{ item }}:{{ pr_sha }}" + tag: "{{ pr_sha }}" + push: true + loop: + - edpm-hardened-uefi + - ironic-python-agent + + - name: Set build images output + ansible.builtin.set_fact: + cifmw_build_images_output: + images: + edpm-hardened-uefi: + image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/edpm-hardened-uefi:{{ pr_sha }}" + ironic-python-agent: + image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/ironic-python-agent:{{ pr_sha }}" + cacheable: true + + - name: Set build images output when EDPM image is not built + when: "'edpm-image-builder' not in zuul_change_list" + ansible.builtin.set_fact: + cifmw_build_images_output: {} + + - name: Set var for cifmw_operator_build_operators var + # It handles the case of setting image_base for + # openstack-ansibleee-operator and openstack-operator project + # for openstack-ansibleee-operator, it will return openstack-ansibleee + # and for openstack-operator, openstack will be returned + when: + - zuul is defined + - "'project' in zuul" + - "'short_name' in zuul.project" + - "'operator' in zuul.project.short_name" + ansible.builtin.set_fact: + cifmw_operator_build_operators: + - name: "openstack-operator" + src: "~/src/github.com/{{ cifmw_operator_build_org }}/openstack-operator" + image_base: >- + {{ zuul.project.short_name | split('-') | reject('search','operator') | join('-') }} + + - name: Build Operators + ansible.builtin.include_role: + name: operator_build + + - name: Get the containers list from container registry + ansible.builtin.uri: + url: "http://{{ cifmw_rp_registry_ip }}:5001/v2/_catalog" + return_content: true + register: cp_imgs + + - name: Add the container list to file + ansible.builtin.copy: + content: "{{ cp_imgs.content }}" + dest: "{{ ansible_user_dir }}/local_registry.log" + mode: "0644" + +- name: Run log related tasks + ansible.builtin.import_playbook: >- + {{ + [ + ansible_user_dir, + zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir, + 'playbooks', + '99-logs.yml' + ] | ansible.builtin.path_join + }} diff --git a/ci/playbooks/meta_content_provider/run.yml b/ci/playbooks/meta_content_provider/run.yml new file mode 100644 index 0000000000..3f9043e2c6 --- /dev/null +++ b/ci/playbooks/meta_content_provider/run.yml @@ -0,0 +1,61 @@ +--- +- name: "Run ci/playbooks/meta_content_provider/run.yml" + hosts: "{{ cifmw_zuul_target_host | default('all') }}" + gather_facts: true + tasks: + - name: Filter out host if needed + when: + - cifmw_zuul_target_host is defined + - cifmw_zuul_target_host != 'all' + - inventory_hostname != cifmw_zuul_target_host + ansible.builtin.meta: end_host + + - name: Deploy Meta content provider + environment: + ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg" + ansible.builtin.command: + chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework" + cmd: >- + ansible-playbook ci/playbooks/meta_content_provider/meta_content_provider.yml + -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml" + -e @scenarios/centos-9/base.yml + -e @scenarios/centos-9/meta_content_provider.yml + -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}" + -e "cifmw_rp_registry_port=5001" + {%- if cifmw_extras is defined %} + {%- for extra_vars in cifmw_extras %} + -e "{{ extra_vars }}" + {%- endfor %} + {%- endif %} + -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml" + + - name: Include inner ansible vars file + ansible.builtin.slurp: + src: "{{ cifmw_artifacts_basedir }}/artifacts/ansible-vars.yml" + register: _inner_ansible + + - name: Return Zuul Data + ansible.builtin.debug: + msg: >- + Running Content provider registry on + {{ cifmw_rp_registry_ip | default('nowhere') }} + + - name: Return data for dependent job + vars: + _inner_ansible_vars: "{{ _inner_ansible.content | b64decode | from_yaml }}" + _dlrn_md5: "{{ _inner_ansible_vars.cifmw_repo_setup_full_hash }}" + _tcib_registry: >- + {%- if _inner_ansible_vars.content_provider_os_registry_url is defined -%} + {{ _inner_ansible_vars.content_provider_os_registry_url }} + {%- else -%} + null + {%- endif -%} + zuul_return: + data: + zuul: + pause: true + content_provider_registry_ip: "{{ cifmw_rp_registry_ip | default('nowhere') | trim }}" + cifmw_operator_build_output: "{{ _inner_ansible_vars.cifmw_operator_build_output }}" + cifmw_build_images_output: "{{ _inner_ansible_vars.cifmw_build_images_output }}" + content_provider_dlrn_md5_hash: "{{ _dlrn_md5 | default('') }}" + content_provider_os_registry_url: "{{ _tcib_registry | trim }}" diff --git a/ci/templates/projects.yaml b/ci/templates/projects.yaml index f3f85feafb..374c0c8892 100644 --- a/ci/templates/projects.yaml +++ b/ci/templates/projects.yaml @@ -21,4 +21,5 @@ - cifmw-multinode-kuttl - cifmw-tcib - cifmw-architecture-validate-hci + - ci-framework-openstack-meta-content-provider # Start generated content diff --git a/roles/build_openstack_packages/tasks/main.yml b/roles/build_openstack_packages/tasks/main.yml index a8bde4a5c8..a20e4664a0 100644 --- a/roles/build_openstack_packages/tasks/main.yml +++ b/roles/build_openstack_packages/tasks/main.yml @@ -20,8 +20,5 @@ - name: Parse Zuul changes and Build RPM packages ansible.builtin.import_tasks: parse_and_build_pkgs.yml -- name: Create repo - ansible.builtin.import_tasks: create_repo.yml - - name: Cleanup dlrn ansible.builtin.import_tasks: cleanup_dlrn.yml diff --git a/roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml b/roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml index 3965742fdb..92352610b7 100644 --- a/roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml +++ b/roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml @@ -50,3 +50,7 @@ branch: "{{ cifmw_bop_openstack_release }}" project: "{{ cifmw_bop_openstack_project_path | basename }}" ansible.builtin.include_tasks: run_dlrn.yml + +- name: Create repo + when: cifmw_bop_change_list | length > 0 + ansible.builtin.import_tasks: create_repo.yml diff --git a/roles/edpm_prepare/tasks/kustomize_and_deploy.yml b/roles/edpm_prepare/tasks/kustomize_and_deploy.yml index 7768195c9b..ae3be6ff74 100644 --- a/roles/edpm_prepare/tasks/kustomize_and_deploy.yml +++ b/roles/edpm_prepare/tasks/kustomize_and_deploy.yml @@ -74,6 +74,16 @@ ).metadata.name }} +- name: Set vars related to update_containers content provider + when: + - content_provider_os_registry_url is defined + - content_provider_os_registry_url != 'null' + ansible.builtin.set_fact: + cifmw_update_containers_registry: "{{ content_provider_os_registry_url | split('/') | first }}" + cifmw_update_containers_org: "{{ content_provider_os_registry_url | split('/') | last }}" + cifmw_update_containers_tag: "{{ content_provider_dlrn_md5_hash }}" + cifmw_update_containers_openstack: true + - name: Update Openstack containers or BM CSV or Ansibleee CSV to update proper image when: >- (cifmw_update_containers_edpm_image_url is defined) or diff --git a/roles/edpm_prepare/tasks/main.yml b/roles/edpm_prepare/tasks/main.yml index 12afd350d4..76b8e1c63f 100644 --- a/roles/edpm_prepare/tasks/main.yml +++ b/roles/edpm_prepare/tasks/main.yml @@ -134,7 +134,9 @@ ansible.builtin.set_fact: cifmw_update_containers_edpm_image_url: "{{ cifmw_build_images_output['images']['edpm-hardened-uefi']['image'] }}" cacheable: true - when: cifmw_build_images_output is defined + when: + - cifmw_build_images_output is defined + - cifmw_build_images_output.keys() | length > 0 # Prepare and kustomize the OpenStackControlPlane CR - name: Prepare OpenStack control plane CR diff --git a/scenarios/centos-9/meta_content_provider.yml b/scenarios/centos-9/meta_content_provider.yml new file mode 100644 index 0000000000..2904205a5a --- /dev/null +++ b/scenarios/centos-9/meta_content_provider.yml @@ -0,0 +1,24 @@ +--- +ansible_user_dir: "{{ lookup('env', 'HOME') }}" +cifmw_installyamls_repos: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/install_yamls" + +# build_operators vars +cifmw_operator_build_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" +cifmw_operator_build_push_org: "openstack-k8s-operators" +cifmw_operator_build_org: "openstack-k8s-operators" +cifmw_operator_build_meta_build: true +cifmw_operator_build_local_registry: 1 +cifmw_operator_build_push_registry_tls_verify: false + +# build_containers vars +cifmw_build_containers_tcib_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/tcib" +cifmw_repo_setup_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/repo-setup" +cifmw_build_containers_repo_dir: "{{ cifmw_basedir }}/artifacts/repositories" +cifmw_build_containers_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" +cifmw_build_containers_push_containers: false +cifmw_build_containers_buildah_push: true +cifmw_build_containers_registry_namespace: podified-antelope-centos9 +cifmw_build_containers_image_tag: "{{ cifmw_repo_setup_full_hash }}" + +# edpm_build_images vars +cifmw_edmp_build_images_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" diff --git a/zuul.d/content_provider.yaml b/zuul.d/content_provider.yaml index d4047a39cd..389dd347e0 100644 --- a/zuul.d/content_provider.yaml +++ b/zuul.d/content_provider.yaml @@ -2,3 +2,24 @@ - job: name: content-provider-base parent: openstack-k8s-operators-content-provider + +- job: + name: openstack-meta-content-provider + parent: openstack-k8s-operators-content-provider + description: | + A zuul job to build contents(rpms, openstack services containers, + operators) from opendev and github changes. + timeout: 4000 + run: + - ci/playbooks/meta_content_provider/run.yml + +- job: + name: ci-framework-openstack-meta-content-provider + parent: openstack-meta-content-provider + files: + - ^roles/build_containers/(?!meta|README).* + - ^roles/build_openstack_packages/(?!meta|README).* + - ^roles/registry_deploy/(?!meta|README).* + - ^roles/edpm_build_images/(?!meta|README).* + - ^roles/operator_build/(?!meta|README).* + - ^ci/playbooks/meta_content_provider/.* diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 1b51102754..d0ef1b87cd 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -12,6 +12,7 @@ - cifmw-multinode-kuttl - cifmw-tcib - cifmw-architecture-validate-hci + - ci-framework-openstack-meta-content-provider - cifmw-molecule-artifacts - cifmw-molecule-build_containers - cifmw-molecule-build_openstack_packages