Automation for ocp-cluster-etcd-operator

examples/all.yaml

@@ -423,3 +423,18 @@ db_volume_size: "10Gi"
 fs_volume_size: "100Gi"
 iso_url: ""
 root_fs_url: ""
+
+# ocp-cluster-etcd-operator vars
+cluster_etcd_enabled: false
+etcd_namespace: "openshift-etcd"
+install_acme_air: false
+run_e2e: false
+install_kube_burner: false
+
+# ocp-kubr-burner vers
+enable_kube_burner: false
+kube_burner_repo: "https://github.com/kube-burner/kube-burner.git"
+kube_burner_dir: "/root/kube-burner"
+kube_burner_path: "/root/kube-burner/examples/workloads"
+kube_burner_version: "1.10.4"
+

examples/ocp_cluster_etcd_operator_vars.yaml

@@ -0,0 +1,7 @@
+## ocp_cluster_etcd-operator vars
+cluster_etcd_enabled: false
+etcd_namespace: openshift-etcd
+install_acme_air: false
+run_e2e: false
+install_kube_burner: false
+

examples/ocp_kube_burner_vars.yaml

@@ -0,0 +1,7 @@
+# ocp-kube-burner vars
+enable_kube_burner: false
+kube_burner_repo: "https://github.com/kube-burner/kube-burner.git"
+kube_burner_dir: "/root/kube-burner"
+kube_burner_path: "/root/kube-burner/examples/workloads"
+kube_burner_version: "1.10.4"
+

playbooks/main.yml

@@ -123,3 +123,10 @@
   when: >
         (hypershift_agent_install is defined and hypershift_agent_install) or
         (hypershift_agent_destroy is defined and hypershift_agent_destroy)
+
+- import_playbook: ocp-cluster-etcd-operator.yml
+  when: cluster_etcd_enabled is defined and cluster_etcd_enabled
+
+- import_playbook: ocp-kube-burner.yml
+  when: enable_kube_burner is defined and enable_kube_burner
+

playbooks/ocp-cluster-etcd-operator.yml

@@ -0,0 +1,7 @@
+---
+
+- name: OCP cluster etcd operator verification
+  hosts: bastion
+  roles: 
+  - ocp-cluster-etcd-operator
+

playbooks/ocp-kube-burner.yml

@@ -0,0 +1,7 @@
+---
+
+- name: OCP kube burner tests
+  hosts: bastion
+  roles: 
+  - ocp-kube-burner
+

playbooks/roles/ocp-cluster-etcd-operator/README.md

@@ -0,0 +1,57 @@
+ocp-cluster-etcd operator
+=========================
+
+The control plane hardware speed values are "Standard", "Slower", or the default, which is "".
+The default setting allows the system to decide which speed to use. This value enables upgrades from versions where this feature does not exist, as the system can select values from previous versions
+
+By selecting one of the other values, you are overriding the default. If you see many leader elections due to timeouts or missed heartbeats and your system is set to "" or "Standard", set the hardware speed to "Slower" to make the system more tolerant to the increased latency
+
+This playbook will handle below:
+- It will check the DaemonSet for HEARTBEAT_INTERVAL and LEADER_ELECTION_TIMEOUT parameters
+- Change the control plane hardware speed tolerance to "Slower"
+- Run the acme-air workload and e2e conformance test suite
+- Also it will run the kube-burner workload test
+
+Note -
+Need to set the vars for ocp-e2e role to enable/run the e2e conformance test suite
+
+Requirements
+------------
+
+ - Running OCP 4.x cluster # better on PowerVS
+
+Role Variables
+--------------
+| Variable                       | Required | Default     | Comments                                       |
+|--------------------------------|----------|-------------|------------------------------------------------|
+| cluster_etcd_enabled           | no       |    false    | Set it to true to run this playbook            |
+| etcd_namespace                 | no       |    false    | cluster etcd namespace.                        |
+| install_acme_air               | no       |    false    | Set to true to deploy acme-air                 |
+| run_e2e                        | no       |    false    | Set to true to run e2e conformance test suite  |
+| install_kube_burner            | no       |    false    | Set to true to install kube-burner workload    |
+
+
+Dependencies
+------------
+
+ - Role ocp-e2e  # to run e2e conformance test suite
+ - Role ocp-kube-burner  #to run kubr-burner tests
+
+Example Playbook
+----------------
+
+```
+  - name: Verify cluster etcd operator
+    include_role:
+      name: ocp-cluster-etcd-operator
+```
+
+License
+-------
+
+See LICENCE.txt
+
+Author Information
+------------------
+
+[email protected]

playbooks/roles/ocp-cluster-etcd-operator/defaults/main.yaml

@@ -0,0 +1,7 @@
+## ocp_cluster_etcd-operator vars
+cluster_etcd_enabled: false
+etcd_namespace: openshift-etcd
+install_acme_air: false
+run_e2e: false
+install_kube_burner: false
+

playbooks/roles/ocp-cluster-etcd-operator/tasks/main.yaml

@@ -0,0 +1,75 @@
+# tasks file for playbooks/roles/ocp-cluster-etcd-operator
+
+# Cluster health check
+- name: Check if cluster operators and nodes are healthy
+  include_role:
+    name: check-cluster-health
+
+# Verify tiers, if cluster is on PowerVS
+- name: Verify tiers
+  shell: oc get sc | grep tier
+  register: tier_output
+  failed_when: tier_output.stdout_lines|length == 0
+
+# Check hardware speed and change the profile to slower if not already
+- name: Verify if Control Plane Hardware Speed profile is "Slower"
+  shell: oc describe etcd/cluster | grep -i slower
+  register: speed_check
+  ignore_errors: true
+
+- name: Update Control Plane Hardware Speed profile to "Slower"
+  shell: |
+    oc patch etcd cluster --type=merge --patch '{"spec": { "controlPlaneHardwareSpeed": "Slower" }}'
+  when: speed_check.rc != 0
+
+- name: Wait for changes to take effect
+  wait_for:
+    timeout: 180
+  when: speed_check.rc != 0
+
+- name: Retrieve the etcd pod name
+  shell: oc get pods -n openshift-etcd --selector=app=etcd --output=jsonpath='{.items[0].metadata.name}'
+  register: etcd_pod_name
+
+- name: Extract HEARTBEAT_INTERVAL and ELECTION_TIMEOUT values
+  shell: |
+    oc describe pod {{ etcd_pod_name.stdout }} -n {{ etcd_namespace }} | grep -e HEARTBEAT_INTERVAL -e ELECTION_TIMEOUT | awk '{print $2}'
+  register: pod_values
+  failed_when:
+    - "'500' not in pod_values.stdout"
+    - "'2500' not in pod_values.stdout"
+
+## Run e2e conformance test suite
+- name: Run e2e conformance test suite
+  include_role:
+    name: ocp-e2e
+  when: run_e2e
+
+# Run the acme-air workload
+- name: Deploy Acmeair Mainservice Java
+  block:
+    - name: Clone acmeair-mainservice-java repo
+      git:
+        repo: https://github.com/ocp-power-demos/acmeair-mainservice-java.git
+        dest: /root/acmeair-mainservice-java/
+
+    - name: Deploy acmeair-mainservice-java pods
+      script: /root/acmeair-mainservice-java/scripts/deployToOpenshift.sh
+      args:
+        chdir: /root/acmeair-mainservice-java
+      register: deployment_output
+
+    - name: Wait for all pods to be ready in acme-air namespace
+      command: >
+        kubectl wait --all --namespace=acme-air --for=condition=Ready pods --timeout=300s
+      register: wait_output
+      failed_when: "'error' in wait_output.stderr.lower() or wait_output.rc != 0"
+      changed_when: False
+  when: install_acme_air
+
+# Run kube burner test
+- name: Run kube-burner-test
+  include_role:
+    name: ocp-kube-burner
+  when: install_kube_burner
+

playbooks/roles/ocp-kube-burner/README.md

@@ -0,0 +1,39 @@
+ocp-kube-burner
+=========================
+This playbook will run the kube-burner workload test
+
+Requirements
+------------
+
+ - Running OCP 4.x cluster
+
+Role Variables
+--------------
+| Variable                       | Required | Default     | Comments                                       |
+|--------------------------------|----------|-------------|------------------------------------------------|
+| enable_kube_burner             | no       |    false    | Set it to true to run this playbook            |
+| kube_burner_repo               | no       |    false    | https://github.com/kube-burner/kube-burner.git |
+| kube_burner_dir                | no       |    false    | /root/kube-burner                              |
+| kube_burner_path               | no       |    false    | /root/kube-burner/examples/workloads           |
+| kube_burner_version            | no       |    false    | set the required version                       |
+
+
+
+Example Playbook
+----------------
+
+```
+  - name: Run kube burner tests
+    include_role:
+      name: ocp-kube-burner
+```
+
+License
+-------
+
+See LICENCE.txt
+
+Author Information
+------------------
+
+[email protected]

playbooks/roles/ocp-kube-burner/defaults/main.yaml

@@ -0,0 +1,7 @@
+## ocp_kube_burner vars
+enable_kube_burner: false
+kube_burner_repo: "https://github.com/kube-burner/kube-burner.git"
+kube_burner_dir: "/root/kube-burner"
+kube_burner_path: "/root/kube-burner/examples/workloads"
+kube_burner_version: "1.10.4"
+

playbooks/roles/ocp-kube-burner/files/api-intensive.yaml

@@ -0,0 +1,35 @@
+---
+- name: Read the content of the file
+  slurp:
+    src: "{{ kube_burner_path }}/api-intensive/api-intensive.yml"
+  register: file_content
+
+- name: Decode the file content and extract first 4 lines
+  set_fact:
+    first_four_lines: "{{ (file_content['content'] | b64decode).split('\n')[3:7] }}"
+
+- name: Add '#' to the first 4 lines
+  lineinfile:
+    path: "{{ kube_burner_path }}/api-intensive/api-intensive.yml"
+    regexp: "^{{ item | regex_escape }}"
+    line: "# {{ item }}"
+    backrefs: yes
+    state: present
+  loop: "{{ first_four_lines }}"
+
+- name: Update jobIterations to 100
+  lineinfile:
+    path: "{{ kube_burner_path }}/api-intensive/api-intensive.yml"
+    regexp: '^(\s*)jobIterations:\s*700\s*$'  
+    line: '\1jobIterations: 100' 
+    backrefs: yes
+
+- name: Run the api-intensity workload
+  shell: kube-burner init -c api-intensive.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/api-intensive"
+  register: api_intensive_output
+  failed_when: 
+    - api_intensive_output.rc != 0
+    - "'👋 Exiting kube-burner' not in api_intensive_output.stdout"
+

playbooks/roles/ocp-kube-burner/files/cluster-density.yaml

@@ -0,0 +1,10 @@
+---
+- name: Run the cluster density workload
+  shell: kube-burner init -c cluster-density.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/cluster-density"
+  register: cluster_density_output
+  failed_when: 
+    - cluster_density_output.rc != 0
+    - "'👋 Exiting kube-burner' not in cluster_density_output.stdout"
+

playbooks/roles/ocp-kube-burner/files/crd-scale.yaml

@@ -0,0 +1,28 @@
+---
+- name: Read the content of the file
+  slurp:
+    src: "{{ kube_burner_path }}/crd-scale/crd-scale.yml"
+  register: file_content
+
+- name: Decode the file content and extract first 4 lines
+  set_fact:
+    first_four_lines: "{{ (file_content['content'] | b64decode).split('\n')[3:7] }}"
+
+- name: Add '#' to the first 4 lines
+  lineinfile:
+    path: "{{ kube_burner_path }}/crd-scale/crd-scale.yml"
+    regexp: "^{{ item | regex_escape }}"
+    line: "# {{ item }}"
+    backrefs: yes
+    state: present
+  loop: "{{ first_four_lines }}"
+
+- name: Run the crd scale workload
+  shell: kube-burner init -c crd-scale.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/crd-scale"
+  register: crd_scale_output
+  failed_when: 
+    - crd_scale_output.rc != 0
+    - "'👋 Exiting kube-burner' not in crd_scale_output.stdout"
+

playbooks/roles/ocp-kube-burner/files/kubelet-density-cni-networkpolicy.yaml

@@ -0,0 +1,10 @@
+---
+- name: Run the kubelet-density-cni-networkpolicy workload
+  shell: kube-burner init -c kubelet-density-cni-networkpolicy.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-cni-networkpolicy"
+  register: kube_den_cni_net_output
+  failed_when: 
+    - kube_den_cni_net_output.rc != 0
+    - "'👋 Exiting kube-burner' not in kube_den_cni_net_output.stdout"
+

playbooks/roles/ocp-kube-burner/files/kubelet-density-cni.yaml

@@ -0,0 +1,10 @@
+---
+- name: Run the kubelet-density-cni workload
+  shell: kube-burner init -c kubelet-density-cni.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-cni"
+  register: kubelet_density_cni_output
+  failed_when: 
+    - kubelet_density_cni_output.rc != 0
+    - "'👋 Exiting kube-burner' not in kubelet_density_cni_output.stdout"
+

playbooks/roles/ocp-kube-burner/files/kubelet-density-heavy.yaml

@@ -0,0 +1,10 @@
+---
+- name: Run the kubelet-density-heavy workload
+  shell: kube-burner init -c kubelet-density-heavy.yml
+  args:
+    chdir: "{{ kube_burner_dir }}/examples/workloads/kubelet-density-heavy"
+  register: kube_den_heavy_output
+  failed_when: 
+    - kube_den_heavy_output.rc != 0
+    - "'👋 Exiting kube-burner' not in kube_den_heavy_output.stdout"
+