Skip to content

Commit

Permalink
oci patchs: Platform External and CCM manifests
Browse files Browse the repository at this point in the history
  • Loading branch information
mtulio committed Mar 13, 2023
1 parent d7eb4fe commit 1be432e
Show file tree
Hide file tree
Showing 23 changed files with 624 additions and 30 deletions.
96 changes: 73 additions & 23 deletions docs/guides/installing-agnostic-oci.md
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,21 @@ OCI_COMPARTMENT_ID_IMAGE="<CHANGE_ME:ocid1.compartment.oc1.UUID>"
EOF
source ~/.oci/env

cat <<EOF > ./vars-oci-ha.yaml
cat <<EOF > ~/.openshift/env
export OCP_CUSTOM_RELEASE="docker.io/mtulio/ocp-release:latest"
OCP_RELEASE_413="quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-x86_64"
EOF
source ~/.openshift/env

CLUSTER_NAME=oci-cr3cmo
cat <<EOF > ./vars-oci-ha_${CLUSTER_NAME}.yaml
provider: oci
cluster_name: oci
cluster_name: ${CLUSTER_NAME}
config_cluster_region: us-sanjose-1
#TODO: create compartment validations
#TODO: allow create compartment from a parent
oci_compartment_id: ${OCI_COMPARTMENT_ID}
oci_compartment_id_dns: ${OCI_COMPARTMENT_ID_DNS}
oci_compartment_id_image: ${OCI_COMPARTMENT_ID_IMAGE}
Expand All @@ -169,10 +179,10 @@ config_base_domain: splat-oci.devcluster.openshift.com
config_ssh_key: "$(cat ~/.ssh/id_rsa.pub)"
config_pull_secret_file: "${HOME}/.openshift/pull-secret-latest.json"
config_cluster_version: 4.13.0-ec.3-x86_64
#config_cluster_version: 4.13.0-ec.3-x86_64
version: 4.13.0-ec.3
#config_installer_environment:
# OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "quay.io/openshift-release-dev/ocp-release:4.13.0-ec.4-x86_64"
config_installer_environment:
OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE: "${OCP_CUSTOM_RELEASE}"
controlplane_instance: VM.Standard3.Flex
controlplane_instance_spec:
Expand Down Expand Up @@ -204,7 +214,41 @@ os_mirror_to_oci:
image_type: QCOW2
#image_type: VMDK
## Apply patches to installer manifests (WIP)
# TODO: we must keep the OCI CCM manifests patch more generic
config_patches:
- rm-capi-machines
#- platform-external-kubelet # PROBLEM hangin kubelete (network)
#- platform-external-kcmo
- deploy-oci-ccm
- yaml_patch # working for OCI, but need to know the path
#- line_regex_patch # ideal, but not working as expected
cfg_patch_yaml_patch_specs:
## patch infra object to create External provider
- manifest: /manifests/cluster-infrastructure-02-config.yml
patch: '{"spec":{"platformSpec":{"type":"External","external":{"platformName":"oci"}}},"status":{"platform":"External","platformStatus":{"type":"External","external":{}}}}'
## OCI : Change the namespace from downloaded assets
#- manifest: /manifests/oci-cloud-controller-manager-02.yaml
# patch: '{"metadata":{"namespace":"oci-cloud-controller-manager"}}'
cfg_patch_line_regex_patch_specs:
- manifest: /manifests/oci-cloud-controller-manager-01-rbac.yaml
#search_string: 'namespace: kube-system'
regexp: '^(.*)(namespace\\: kube-system)$'
#line: 'namespace: oci-cloud-controller-manager'
line: '\\1namespace: oci-cloud-controller-manager'
- manifest: /manifests/oci-cloud-controller-manager-02.yaml
regexp: '^(.*)(namespace\\: kube-system)$'
line: '\\1namespace: oci-cloud-controller-manager'
EOF


```

### Install the clients
Expand Down Expand Up @@ -295,6 +339,28 @@ ansible-playbook mtulio.okd_installer.create_node \
oc adm certificate approve $(oc get csr -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name')
```

### Create all

```bash
ansible-playbook mtulio.okd_installer.create_all \
-e certs_max_retries=20 \
-e cert_wait_interval_sec=60 \
-e @./vars-oci-ha.yaml
```

> TO DO: measure total time
## Review the cluster

```bash
export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig

oc get nodes
oc get co
```

## OPCT setup

- Create the OPCT [dedicated] node

> https://redhat-openshift-ecosystem.github.io/provider-certification-tool/user/#option-a-command-line
Expand Down Expand Up @@ -344,30 +410,14 @@ oc adm taint node opct-01.priv.ocp.oraclevcn.com node-role.kubernetes.io/tests="
~/opct/bin/openshift-provider-cert-linux-amd64-v0.3.0 report *.tar.gz
```

## Generate custom image

### Create all

```bash
ansible-playbook mtulio.okd_installer.create_all \
-e @./vars-oci-ha.yaml \
-e certs_max_retries=20 \
-e cert_wait_interval_sec=60
```
> TO DO: measure total time
## Review the cluster

```bash
export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig

oc get nodes
oc get co
```

## Destroy

```bash
ansible-playbook mtulio.okd_installer.destroy_cluster \
-e @./vars-oci-ha.yaml
ansible-playbook mtulio.okd_installer.destroy_cluster -e @./vars-oci-ha.yaml
```
6 changes: 3 additions & 3 deletions playbooks/create_all.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@
vars:
mode: create

- name: OKD Installer | Create all | create stack | IAM
ansible.builtin.import_playbook: stack_iam.yaml

- name: OKD Installer | Create all | os_mirror
ansible.builtin.import_playbook: os_mirror.yaml
when: os_mirror | d(false)

- name: OKD Installer | Create all | create stack | network
ansible.builtin.import_playbook: stack_network.yaml

- name: OKD Installer | Create all | create stack | IAM
ansible.builtin.import_playbook: stack_iam.yaml

- name: OKD Installer | Create all | create stack | DNS
ansible.builtin.import_playbook: stack_dns.yaml

Expand Down
2 changes: 1 addition & 1 deletion playbooks/group_vars/all.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ collection_cluster_dir: "{{ bindir | d(collection_work_dir + '/clusters') }}"

config_install_dir: "{{ collection_cluster_dir }}/{{ cluster_name }}"
bin_openshift_install: "{{ collection_bin_dir }}/openshift-install"
bin_oc: "{{ collection_bin_dir }}/openshift-install"
bin_oc: "{{ collection_bin_dir }}/oc"

## export CONFIG_PULL_SECRET_FILE=${HOME}/.openshift/pull-secret-latest.jso
config_pull_secret_file: "{{ lookup('ansible.builtin.env', 'CONFIG_PULL_SECRET_FILE') }}"
Expand Down
15 changes: 14 additions & 1 deletion playbooks/vars/oci/profiles/ha/node-bootstrap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,21 @@ compute_resources:
# availability_domain: Uocm:PHX-AD-1
agent_config:
are_all_plugins_disabled: true


# Disk Configuration
preserve_boot_volume: false
source_details:
source_type: image
boot_volume_size_in_gbs: 120
boot_volume_vpus_per_gb: 30

# that config will prevent actions like stop/start (not desired)
# preemptible_instance_config:
# preemption_action:
# preserve_boot_volume: false
# type: TERMINATE

# Network
create_vnic_details:
display_name: "{{ cluster_state.infra_id }}-bootstrap-vnic0"
assign_public_ip: true
Expand All @@ -108,6 +117,10 @@ compute_resources:
metadata:
user_data: "{{ lookup('template', 'ocp-bootstrap-user-data.j2') | to_nice_json | string | b64encode }}"

# launch_options:
# firmware: BIOS
# boot_volume_type: PARAVIRTUALIZED

callbacks:
- name: nlb
# nlb_name: ocp-nlb
Expand Down
3 changes: 3 additions & 0 deletions playbooks/vars/oci/profiles/ha/node-compute.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: no
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-worker-01"
region: "{{ config_cluster_region }}"
Expand Down Expand Up @@ -95,6 +96,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: no
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-worker-02"
region: "{{ config_cluster_region }}"
Expand Down Expand Up @@ -155,6 +157,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: no
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-worker-03"
region: "{{ config_cluster_region }}"
Expand Down
3 changes: 3 additions & 0 deletions playbooks/vars/oci/profiles/ha/node-controlplane.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: no
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-master-01"
region: "{{ config_cluster_region }}"
Expand Down Expand Up @@ -94,6 +95,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: no
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-master-02"
region: "{{ config_cluster_region }}"
Expand Down Expand Up @@ -154,6 +156,7 @@ compute_resources:
# OCI spec
spec:
state: present
wait: yes
compartment_id: "{{ oci_compartment_id }}"
display_name: "{{ cluster_state.infra_id }}-master-03"
region: "{{ config_cluster_region }}"
Expand Down
2 changes: 1 addition & 1 deletion roles/cloud_compute
2 changes: 1 addition & 1 deletion roles/cloud_network
30 changes: 30 additions & 0 deletions roles/config/tasks/patches-manifests/deploy-oci-ccm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
---
- name: Patch | Create OCI CCM NS
ansible.builtin.template:
src: patches/oci-ccm-ns.yaml.j2
dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-00-namespace.yaml"
mode: 0644

- name: Patch | Create OCI CCM RBAC
ansible.builtin.template:
src: patches/oci-cloud-controller-manager-rbac.yaml.j2
dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-01-rbac.yaml"
mode: 0644

- name: Patch | Create OCI CCM Secret
ansible.builtin.set_fact:
oci_ccm_secret_data: "{{ lookup('template', 'patches/oci-ccm-secret-data.yaml.j2') | from_yaml }}"

- name: Patch | Create OCI CCM Secret
ansible.builtin.template:
src: patches/oci-ccm-secret.yaml.j2
dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-01-secret.yaml"
mode: 0644
vars:
oci_compartment_id: oci_compartment_id

- name: Patch | Create OCI CCM
ansible.builtin.template:
src: patches/oci-cloud-controller-manager.yaml.j2
dest: "{{ config_install_dir }}/manifests/oci-cloud-controller-manager-02.yaml"
mode: 0644
12 changes: 12 additions & 0 deletions roles/config/tasks/patches-manifests/line_regex_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
- name: Config | Patch manifests | Line regex
ansible.builtin.lineinfile:
path: "{{ config_install_dir }}{{ patch_spec.manifest }}"
# search_string: "{{ patch_spec.search_string }}"
# line: "{{ patch_spec.line }}"
backrefs: true
regexp: "{{ patch_spec.regexp }}"
line: "{{ patch_spec.line }}"
loop: "{{ cfg_patch_line_regex_patch_specs }}"
loop_control:
loop_var: patch_spec
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
- name: Patch | Create KCMO Config for External provider
ansible.builtin.template:
src: patches/cm-kcmo-external.yaml.j2
dest: "{{ config_install_dir }}/manifests/kube-controller-manager-operator-configmap.yaml"
mode: 0644
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
---
- debug: var=bin_oc

- name: get pod sha
ansible.builtin.shell: |
{{ bin_oc }} adm release info \
-a {{ config_pull_secret_file }} \
--image-for='pod' "quay.io/openshift-release-dev/ocp-release:{{ config_cluster_version }}"
environment: "{{ config_installer_environment | d(omit) }}"
register: _cmd_release_info

- debug: var=_cmd_release_info

- ansible.builtin.set_fact:
_cloud_provider_name: external
_pod_image: "{{ _cmd_release_info.stdout }}"
_mc_kubelet_workaround: ""

# _mc_kubelet_workaround: >
# #KUBELET_PROVIDERID=$(curl -H "Authorization: Bearer Oracle" -sL http://169.254.169.254/opc/v2/instance/ | jq -r .id)

- name: Crete kubelet config
ansible.builtin.template:
src: patches/mc-kubelet.yaml.j2
dest: "{{ config_install_dir }}/openshift/99_openshift-machineconfig_02-{{ _machine_config_role }}-kubelet.yaml"
#dest: "/tmp/99_openshift-machineconfig_02-{{ _machine_config_role }}-kubelet.yaml"
loop_control:
loop_var: _machine_config_role
loop:
- master
- worker
22 changes: 22 additions & 0 deletions roles/config/tasks/patches-manifests/yaml_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---

- ansible.builtin.include_tasks: ./yaml_patch_exec.yaml
loop: "{{ cfg_patch_yaml_patch_specs }}"
loop_control:
loop_var: patch_spec

# - name: patch | reading file {{ config_install_dir + cfg_patch_generic_update.filename }}
# set_fact:
# patch_file: "{{ lookup('file', config_install_dir + cfg_patch_generic_update.filename) | from_yaml }}"

# - debug: var=patch_file
# - debug: var=cfg_patch_generic_update.patch

# - name: patch | patching content
# set_fact:
# new_content: "{{ patch_file | combine(cfg_patch_generic_update.patch|from_json, recursive=True) }}"

# - name: patch | saving file {{ config_install_dir + cfg_patch_generic_update.filename }}
# copy:
# dest: "{{ config_install_dir + cfg_patch_generic_update.filename }}"
# content: "{{ new_content | to_nice_yaml(indent=2) }}"
18 changes: 18 additions & 0 deletions roles/config/tasks/patches-manifests/yaml_patch_exec.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
---
# generic update executor

- name: patch | reading file {{ config_install_dir + patch_spec.manifest }}
set_fact:
patch_file: "{{ lookup('file', config_install_dir + patch_spec.manifest) | from_yaml }}"

- debug: var=patch_file
- debug: var=patch_spec.patch

- name: patch | patching content
set_fact:
new_content: "{{ patch_file | combine(patch_spec.patch|from_json, recursive=True) }}"

- name: patch | saving file {{ config_install_dir + patch_spec.manifest }}
copy:
dest: "{{ config_install_dir + patch_spec.manifest }}"
content: "{{ new_content | to_nice_yaml(indent=2) }}"
Loading

0 comments on commit 1be432e

Please sign in to comment.