From b878cf5754cfeef08c74582cd616c2c8ff2162de Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Mon, 13 Feb 2023 15:20:57 -0300 Subject: [PATCH] feat/provider-oci: add full-stack creation --- docs/guides/installing-agnostic-oci.md | 97 +++++-- playbooks/vars/oci/profiles/ha/dns.yaml | 2 +- .../vars/oci/profiles/ha/loadbalancer.yaml | 54 ++-- playbooks/vars/oci/profiles/ha/network.yaml | 270 +++++++++--------- .../vars/oci/profiles/ha/node-bootstrap.yaml | 114 ++++++++ .../oci/profiles/ha/node-controlplane.yaml | 268 +++++++++++++++++ roles/bootstrap/tasks/oci.yaml | 57 ++++ roles/cloud_compute | 2 +- roles/cloud_load_balancer | 2 +- roles/cloud_network | 2 +- 10 files changed, 698 insertions(+), 170 deletions(-) create mode 100644 playbooks/vars/oci/profiles/ha/node-bootstrap.yaml create mode 100644 playbooks/vars/oci/profiles/ha/node-controlplane.yaml create mode 100644 roles/bootstrap/tasks/oci.yaml diff --git a/docs/guides/installing-agnostic-oci.md b/docs/guides/installing-agnostic-oci.md index 535f356..6f90ffb 100644 --- a/docs/guides/installing-agnostic-oci.md +++ b/docs/guides/installing-agnostic-oci.md @@ -1,12 +1,15 @@ # Install OKD/OCP on OCI using agnostic method +> This document is under development. + Install OCP/OKD Cluster on Oracle Cloud Infrastructure using agnostic installation/UPI. ## Prerequisites - ### Setup Ansible project +> This steps should be made only when OCI provider is under development - not merged to `main` branch. Then the normal install flow should be used. + - Setup your ansible workdir (optional, you can use the defaults) ```bash @@ -41,7 +44,7 @@ wget https://raw.githubusercontent.com/mtulio/ansible-collection-okd-installer/m - Update with OCI requirements -``` +```bash cat <> requirements.txt # Oracle Cloud Infrastructure @@ -59,19 +62,21 @@ EOF - Install ansible and dependencies -``` +```bash pip install -r requirements.txt ``` -- Install the collections +- Install the Collections -``` +```bash ansible-galaxy collection install -r requirements.yml ``` - Get the latest (under development) okd-installer for OCI -``` +> https://github.com/mtulio/ansible-collection-okd-installer/pull/26 + +```bash git clone -b feat-add-provider-oci --recursive \ git@github.com:mtulio/ansible-collection-okd-installer.git \ collections/ansible_collections/mtulio/okd_installer @@ -80,7 +85,7 @@ git clone -b feat-add-provider-oci --recursive \ - Check if the collection is present -``` +```bash $ ansible-galaxy collection list |egrep "(okd_installer|^oracle)" mtulio.okd_installer 0.0.0-latest oracle.oci 4.11.0 @@ -110,7 +115,7 @@ ansible localhost \ You must be able to collect the user information. -## okd-installer +## OCP Cluster Setup on OCI ### Generate the vars file @@ -118,11 +123,11 @@ You must be able to collect the user information. cat < ~/.oci/env OCI_COMPARTMENT_ID="" EOF - source ~/.oci/env + cat < ./vars-oci-ha.yaml provider: oci -cluster_name: ocp-oci +cluster_name: mrb config_cluster_region: us-sanjose-1 oci_compartment_id: ${OCI_COMPARTMENT_ID} @@ -144,7 +149,9 @@ compute_instance_spec: cpu_count: 8 memory_gb: 16 +#> TODO extract from stream file # https://rhcos.mirror.openshift.com/art/storage/prod/streams/4.12/builds/412.86.202212081411-0/aarch64/rhcos-412.86.202212081411-0-openstack.aarch64.qcow2.gz +# $ jq -r '.architectures["x86_64"].artifacts.openstack.formats["qcow2.gz"].disk.location' ~/.ansible/okd-installer/clusters/ocp-oci/coreos-stream.json` custom_image_id: rhcos-412.86.202212081411-0-openstack.aarch64.qcow2.gz EOF ``` @@ -192,14 +199,72 @@ ansible-playbook mtulio.okd_installer.stack_loadbalancer \ ### Compute Stack -- Bootstrap -TODO +#### Bootstrap + +- Mirror image (Ansible Role+Playbook Not implemented) -- Control Plane +> TODO: config to mirror from openstack image to OCI -TODO +> Currently the image is download manually, and added to the OCI Console as a image. -- Compute -TODO +Steps to mirror: + +- Get the artifact URL from stream-json +- Create Bucket for images, if not exits +- Upload the image qcow2.gz +- Get the signed URL for the image object +- Create an image from signed URL +- Get the image ID, and set the global var `custom_image_id` + +> `$ jq -r '.architectures["x86_64"].artifacts.openstack.formats["qcow2.gz"].disk.location' ~/.ansible/okd-installer/clusters/ocp-oci/coreos-stream.json` + +```bash +os_mirror: yes +os_mirror_src: stream +os_mirror_stream: + architecture: x86_64 + platform: openstack + format: qcow2.gz + +os_mirror_dest_provider: oci +os_mirror_dest_oci: + compartment_id: + bucket: +``` + +- Upload the bootstrap ignition to blob and Create the Bootstrap Instance + +```bash +ansible-playbook mtulio.okd_installer.create_node \ + -e node_role=bootstrap \ + -e @./vars-oci-ha.yaml +``` + +- Create the Control Plane nodes + +```bash +ansible-playbook mtulio.okd_installer.create_node \ + -e node_role=controlplane \ + -e @./vars-oci-ha.yaml +``` + +- Create the Compute nodes + +> TODO: create instance Pool + +> TODO: Approve certificates (bash loop or use existing playbook) + +``` +oc adm certificate approve $(oc get csr -o json |jq -r '.items[] | select(.status.certificate == null).metadata.name') +``` + +## Review the cluster + +```bash +export KUBECONFIG=${HOME}/.ansible/okd-installer/clusters/${cluster_name}/auth/kubeconfig + +oc get nodes +oc get co +``` diff --git a/playbooks/vars/oci/profiles/ha/dns.yaml b/playbooks/vars/oci/profiles/ha/dns.yaml index 7406f85..d806577 100644 --- a/playbooks/vars/oci/profiles/ha/dns.yaml +++ b/playbooks/vars/oci/profiles/ha/dns.yaml @@ -14,7 +14,7 @@ cloud_dns_zones: # private - name: "{{ cluster_state.dns.cluster_domain }}" provider: oci - view_name: "{{ cluster_state.infra_id }}-vpc" + view_name: "{{ cluster_state.infra_id }}-vcn" spec: scope: PRIVATE zone_type: PRIMARY diff --git a/playbooks/vars/oci/profiles/ha/loadbalancer.yaml b/playbooks/vars/oci/profiles/ha/loadbalancer.yaml index e253bca..d41d705 100644 --- a/playbooks/vars/oci/profiles/ha/loadbalancer.yaml +++ b/playbooks/vars/oci/profiles/ha/loadbalancer.yaml @@ -25,15 +25,15 @@ cloud_load_balancer_provider: oci # OCI NLB: https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-module cloud_loadbalancers: - - name: "{{ cluster_state.infra_id }}-ext" + - name: "{{ cluster_state.infra_id }}-nlb" provider: oci type: network # Is it supported multi-subnets? - subnet_name: "{{ cluster_state.infra_id }}-net-public" + subnet_name: "{{ cluster_state.infra_id }}-net-public-lb" spec: compartment_id: "{{ oci_compartment_id }}" - display_name: "{{ cluster_state.infra_id }}-ext" + display_name: "{{ cluster_state.infra_id }}-nlb" is_private: false is_preserve_source_destination: true nlb_ip_version: IPV4 @@ -44,10 +44,10 @@ cloud_loadbalancers: backend_set: - provider: oci spec: - name: "{{ cluster_state.infra_id }}-aext" + name: "{{ cluster_state.infra_id }}-api" is_preserve_source: no ip_version: IPV4 - #policy: TWO_TUPLE + policy: FIVE_TUPLE #backends: [] health_checker: port: 6443 @@ -59,10 +59,25 @@ cloud_loadbalancers: - provider: oci spec: - name: "{{ cluster_state.infra_id }}-ig-80" + name: "{{ cluster_state.infra_id }}-mcs" is_preserve_source: no ip_version: IPV4 - #policy: TWO_TUPLE + policy: FIVE_TUPLE + #backends: [] + health_checker: + port: 22623 + protocol: HTTPS + return_code: 200 + url_path: /healthz + interval_in_millis: 10000 + timeout_in_millis: 3000 + + - provider: oci + spec: + name: "{{ cluster_state.infra_id }}-ing-http" + is_preserve_source: no + ip_version: IPV4 + policy: FIVE_TUPLE #backends: [] # TCP/31794 health_checker: port: 31261 @@ -74,7 +89,7 @@ cloud_loadbalancers: - provider: oci spec: - name: "{{ cluster_state.infra_id }}-ig-443" + name: "{{ cluster_state.infra_id }}-ing-https" is_preserve_source: no ip_version: IPV4 #policy: TWO_TUPLE @@ -90,22 +105,29 @@ cloud_loadbalancers: # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_load_balancer_listener_module.html#ansible-collections-oracle-oci-oci-network-load-balancer-listener-module listeners: - spec: - name: "{{ cluster_state.infra_id }}-aext" - default_backend_set_name: "{{ cluster_state.infra_id }}-aext" + name: "{{ cluster_state.infra_id }}-api" + default_backend_set_name: "{{ cluster_state.infra_id }}-api" ip_version: IPV4 port: 6443 protocol: TCP - spec: - name: "{{ cluster_state.infra_id }}-ig-80" - default_backend_set_name: "{{ cluster_state.infra_id }}-ig-80" + name: "{{ cluster_state.infra_id }}-mext" + default_backend_set_name: "{{ cluster_state.infra_id }}-mcs" + ip_version: IPV4 + port: 22623 + protocol: TCP + + - spec: + name: "{{ cluster_state.infra_id }}-ing-http" + default_backend_set_name: "{{ cluster_state.infra_id }}-ing-http" ip_version: IPV4 port: 80 protocol: TCP - spec: - name: "{{ cluster_state.infra_id }}-ig-443" - default_backend_set_name: "{{ cluster_state.infra_id }}-ig-443" + name: "{{ cluster_state.infra_id }}-ing-https" + default_backend_set_name: "{{ cluster_state.infra_id }}-ing-https" ip_version: IPV4 port: 443 protocol: TCP @@ -138,7 +160,7 @@ cloud_loadbalancers: # private address - name: register_dns rr_ip: private - view_name: "{{ cluster_state.infra_id }}-vpc" + view_name: "{{ cluster_state.infra_id }}-vcn" spec: zone_name_or_id: "{{ cluster_state.dns.cluster_domain }}" compartment_id: "{{ oci_compartment_id }}" @@ -150,7 +172,7 @@ cloud_loadbalancers: - name: register_dns rr_ip: public - view_name: "{{ cluster_state.infra_id }}-vpc" + view_name: "{{ cluster_state.infra_id }}-vcn" spec: zone_name_or_id: "{{ cluster_state.dns.cluster_domain }}" compartment_id: "{{ oci_compartment_id }}" diff --git a/playbooks/vars/oci/profiles/ha/network.yaml b/playbooks/vars/oci/profiles/ha/network.yaml index 0af104f..2c4285b 100644 --- a/playbooks/vars/oci/profiles/ha/network.yaml +++ b/playbooks/vars/oci/profiles/ha/network.yaml @@ -6,85 +6,11 @@ ######################### # TODO: fix those rules to more restrictive. This is used to dev env. -security_groups: - - name: "{{ cluster_state.infra_id }}-bootstrap-sg" - description: Bootstrap Security Group - purge_tags: no - tags: "{{ cluster_state.tags | combine({'Name': cluster_state.infra_id + '-bootstrap-sg'}) }}" - purge_rules: true - rules: - - proto: tcp - from_port: 22 - to_port: 22 - cidr_ip: 0.0.0.0/0 - - proto: all - cidr_ip: 10.0.0.0/8 - purge_rules_egress: true - rules_egress: - - proto: all - cidr_ip: 0.0.0.0/0 - - - name: "{{ cluster_state.infra_id }}-controlplane-sg" - description: Security Group for Control Plane nodes - purge_tags: no - tags: "{{ cluster_state.tags | combine({'Name': cluster_state.infra_id + '-controlplane-sg'}) }}" - purge_rules: true - rules: - - proto: all - group_name: "{{ cluster_state.infra_id }}-controlplane-sg" - - proto: all - group_name: "{{ cluster_state.infra_id }}-compute-sg" - group_desc: Security Group for Compute nodes - - proto: all - cidr_ip: 10.0.0.0/8 - - purge_rules_egress: true - rules_egress: - - proto: all - cidr_ip: 0.0.0.0/0 - - - name: "{{ cluster_state.infra_id }}-compute-sg" - description: Security Group for Compute nodes - purge_tags: no - tags: "{{ cluster_state.tags | combine({'Name': cluster_state.infra_id + '-compute-sg'}) }}" - purge_rules: true - rules: - - proto: all - group_name: "{{ cluster_state.infra_id }}-controlplane-sg" - group_desc: Security Group for Control Plane nodes - - proto: all - group_name: "{{ cluster_state.infra_id }}-compute-sg" - - proto: all - cidr_ip: 10.0.0.0/8 - - purge_rules_egress: true - rules_egress: - - proto: all - cidr_ip: 0.0.0.0/0 - - - name: "{{ cluster_state.infra_id }}-vpce-ec2" - description: Security Group for EC2 VPC Endpoint - purge_tags: no - tags: "{{ cluster_state.tags | combine({'Name': cluster_state.infra_id + '-vpce-ec2'}) }}" - purge_rules: true - rules: - - proto: tcp - cidr_ip: 10.0.0.0/8 - ports: - - 443 - rule_desc: allow VPC CIDR on port 443 - - purge_rules_egress: true - rules_egress: - - proto: tcp - cidr_ip: 0.0.0.0/0 - ports: - - 443 - rule_desc: allow VPC CIDR on port 443 +# security_groups: [] cloud_networks: ## OCI US San Jose 1 (HA topology) - - name: "{{ cluster_state.infra_id }}-vpc" + - name: "{{ cluster_state.infra_id }}-vcn" block: "{{ okd_net_default_cidr }}" provider: oci region: "{{ config_cluster_region }}" @@ -94,13 +20,13 @@ cloud_networks: tags: "{{ cluster_state.tags | d({}) }}" spec_vcn: - dns_label: ocpinstaller + dns_label: ocp internet_gateway: spec: is_enabled: yes display_name: "{{ cluster_state.infra_id }}-igw" - # carrier_gateway: true + # nat_gateways: # - name: "{{ cluster_state.infra_id }}-natgw-1a" # subnet: "{{ cluster_state.infra_id }}-net-public-1a" @@ -116,27 +42,19 @@ cloud_networks: # wait: true route_tables: - # - name: "{{ cluster_state.infra_id }}-rt-private-1a" - # routes: - # - dest: 0.0.0.0/0 - # gw_type: natgw - # target: "{{ cluster_state.infra_id }}-natgw-1a" - - # - name: "{{ cluster_state.infra_id }}-rt-private-1b" - # routes: - # - dest: 0.0.0.0/0 - # gw_type: natgw - # target: "{{ cluster_state.infra_id }}-natgw-1b" - - # - name: "{{ cluster_state.infra_id }}-rt-private-1c" - # routes: - # - dest: 0.0.0.0/0 - # gw_type: natgw - # target: "{{ cluster_state.infra_id }}-natgw-1c" + - public: yes + spec: + display_name: "{{ cluster_state.infra_id }}-rt-public-lb" + route_rules: + - spec: + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + network_entity_type: internet-gateway + network_entity_name: "{{ cluster_state.infra_id }}-igw" - public: yes spec: - display_name: "{{ cluster_state.infra_id }}-rt-public" + display_name: "{{ cluster_state.infra_id }}-rt-public-nodes" route_rules: - spec: destination: "0.0.0.0/0" @@ -146,20 +64,67 @@ cloud_networks: # - public: no # spec: - # display_name: "{{ cluster_state.infra_id }}-rt-private" + # display_name: "{{ cluster_state.infra_id }}-rt-private-app" # route_rules: # - spec: # destination: "0.0.0.0/0" # destination_type: "CIDR_BLOCK" - # network_entity_type: internet-gateway + # network_entity_type: nat-gateway # network_entity_name: "{{ cluster_state.infra_id }}-natgw" - # - name: "{{ cluster_state.infra_id }}-rt-public-edge" - # routes: - # - dest: 0.0.0.0/0 - # gw_type: cagw + + # create permissive security Lists to force to use NSGs + # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_security_list_module.html#ansible-collections-oracle-oci-oci-network-security-list-module + security_lists: + - spec: + display_name: "{{ cluster_state.infra_id }}-seclist-default" + ingress_security_rules: + - protocol: all + source: 0.0.0.0/0 + is_stateless: false + source_type: CIDR_BLOCK + description: Allow all inbound + egress_security_rules: + - destination: 0.0.0.0/0 + protocol: all + destination_type: CIDR_BLOCK + is_stateless: false + description: Allow All Outbound + + - spec: + display_name: "{{ cluster_state.infra_id }}-seclist-lb" + # TODO restrict only for I/O of listeners/backendSet + ingress_security_rules: + - protocol: all + source: 0.0.0.0/0 + is_stateless: false + source_type: CIDR_BLOCK + description: Allow all inbound + egress_security_rules: + - destination: "10.0.0.0/16" + protocol: 6 + tcp_options: + destination_port_range: + min: 6443 + max: 6443 + destination_type: CIDR_BLOCK + is_stateless: false + description: Allow API Outbound + - destination: "10.0.0.0/16" + protocol: 6 + tcp_options: + destination_port_range: + min: 22623 + max: 22623 + destination_type: CIDR_BLOCK + is_stateless: false + description: Allow MCS Outbound subnets: + # Best practice create dedicated LB Subnet/Rtb? + # https://docs.oracle.com/en-us/iaas/Content/GSG/Tasks/loadbalancing.htm#Update + # Your load balancer must reside in different subnets from your application instances. This configuration allows you to keep your application instances secured in subnets with stricter access rules, while allowing public internet traffic to the load balancer in the public subnets. + # - public: no # spec: # display_name: "{{ cluster_state.infra_id }}-net-private" @@ -167,37 +132,74 @@ cloud_networks: # prohibit_public_ip_on_vnic: true - public: yes + route_table_name: "{{ cluster_state.infra_id }}-rt-public-nodes" + security_list_names: + - "{{ cluster_state.infra_id }}-seclist-default" spec: cidr_block: "10.0.16.0/22" - display_name: "{{ cluster_state.infra_id }}-net-public" - - # az: us-east-1a - # cidr: 10.0.16.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-public" - # map_public: true - # - name: "{{ cluster_state.infra_id }}-net-public-1b" - # az: us-east-1b - # cidr: 10.0.20.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-public" - # map_public: true - # - name: "{{ cluster_state.infra_id }}-net-public-1c" - # az: us-east-1c - # cidr: 10.0.24.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-public" - # map_public: true - - # - name: "{{ cluster_state.infra_id }}-net-private-1a" - # az: us-east-1a - # cidr: 10.0.48.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-private-1a" - # map_public: false - # - name: "{{ cluster_state.infra_id }}-net-private-1b" - # az: us-east-1b - # cidr: 10.0.52.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-private-1b" - # map_public: false - # - name: "{{ cluster_state.infra_id }}-net-private-1c" - # az: us-east-1c - # cidr: 10.0.56.0/22 - # route_table: "{{ cluster_state.infra_id }}-rt-private-1c" - # map_public: false + display_name: "{{ cluster_state.infra_id }}-net-public-nodes" + dns_label: pub + + # - public: no + # security_list_names: + # - "{{ cluster_state.infra_id }}-seclist-default" + # spec: + # cidr_block: "10.0.32.0/22" + # display_name: "{{ cluster_state.infra_id }}-net-private-nodes" + # prohibit_internet_ingress: true + # prohibit_public_ip_on_vnic: true + # dns_label: nodes + + - public: yes + route_table_name: "{{ cluster_state.infra_id }}-rt-public-lb" + security_list_names: + - "{{ cluster_state.infra_id }}-seclist-lb" + spec: + cidr_block: "10.0.0.0/24" + display_name: "{{ cluster_state.infra_id }}-net-public-lb" + + + # https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_network_security_list_module.html#ansible-collections-oracle-oci-oci-network-security-list-module + network_security_groups: + - spec: + display_name: "{{ cluster_state.infra_id }}-nsg-bootstrap" + rules: + spec: + security_rules: + # INGRESS + - description: allow all inbound subnet + source: "10.0.0.0/16" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: all + + - description: allow ssh + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 22 + max: 22 + + - description: allow API + source: "0.0.0.0/0" + source_type: "CIDR_BLOCK" + direction: INGRESS + is_stateless: false + protocol: 6 + tcp_options: + destination_port_range: + min: 6443 + max: 6443 + + # EGRESS + - description: allow all outbound subnet + destination: "0.0.0.0/0" + destination_type: "CIDR_BLOCK" + direction: EGRESS + is_stateless: false + protocol: all diff --git a/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml b/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml new file mode 100644 index 0000000..15bfda2 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-bootstrap.yaml @@ -0,0 +1,114 @@ +--- +_cluster_prefix: "{{ cluster_state.infra_id }}" + +# Vars used on Bootstrap +bootstrap_bucket: "{{ _cluster_prefix }}-infra" + +# Vars used on Machine/Compute Stack +_instance_type: "{{ bootstrap_instance | d('m6i.xlarge') }}" +_instance_profile: "{{ cluster_state.compute.iam_profile_bootstrap }}" +# _image_id: "{{ custom_image_id | d(cluster_state.compute.image_id) }}" +_image_id: "rhcos-412.86.202212081411-0-openstack.x86_64" +_subnet_name: "{{ _cluster_prefix }}-net-public-1a" + +_machine_suffix: '' + +## User Data template +openshift_userdata: + # config_source: "{{ bootstrap_bucket_signed_url }}" + config_source: "{{ bootstrap_bucket_signed_url }}" + +## Common vars used in the Stack vars +# _common: +# prefix: "{{ _cluster_prefix }}-bootstrap" +# detailed_monitoring: yes +# ebs_optimized: no +# image_id: "{{ _image_id }}" +# instance_role: "{{ _instance_profile }}" +# instance_type: "{{ _instance_type }}" +# security_groups: +# - "{{ _cluster_prefix }}-bootstrap-sg" +# - "{{ _cluster_prefix }}-controlplane-sg" +# state: present +# tags: "{{ cluster_state.tags }}" +# termination_protection: no +# volumes: +# - device_name: /dev/xvda +# ebs: +# volume_size: 128 +# volume_type: gp3 +# delete_on_termination: true +# - device_name: /dev/xvdd +# ebs: +# volume_size: 32 +# volume_type: gp3 +# delete_on_termination: true + +# vpc_subnet_name: "{{ _subnet_name }}" +# wait: yes +# wait_timeout: 500 + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + - provider: oci + type: machine + # name: "{{ cluster_state.infra_id }}-bootstrap{{ _machine_suffix }}" + + # cloud_compute role to discovery for OCI: + image_name: rhcos-412.86.202212081411-0-openstack.x86_64 + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public-nodes" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-bootstrap" + # OCI spec + spec: + state: present + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-bootstrap{{ _machine_suffix }}" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + fault_domain: FAULT-DOMAIN-1 + # availability_domain: Uocm:PHX-AD-1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-bootstrap-vnic0" + assign_public_ip: true + assign_private_dns_record: true + hostname_label: "bootstrap{{ _machine_suffix }}" + # defined_tags: {'Operations': {'CostCenter': 'US'}} + # freeform_tags: {'Department': 'Finance'} + # private_ip: private_ip_example + # skip_source_dest_check: true + # vlan_id: "ocid1.vlan.oc1..xxxxxxEXAMPLExxxxxx" + #subnet_id: "{{ machine_subnet_id }}" + #nsg_ids: "{{ machine_nsg_ids }}" + metadata: + user_data: "{{ lookup('template', 'ocp-bootstrap-user-data.j2') | to_nice_json | string | b64encode }}" + + callbacks: + - name: nlb + # nlb_name: ocp-nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + # - name: "6443" + # port: 6443 + # - name: "22623" + # port: 22623 diff --git a/playbooks/vars/oci/profiles/ha/node-controlplane.yaml b/playbooks/vars/oci/profiles/ha/node-controlplane.yaml new file mode 100644 index 0000000..5a9cb74 --- /dev/null +++ b/playbooks/vars/oci/profiles/ha/node-controlplane.yaml @@ -0,0 +1,268 @@ +--- +_cluster_prefix: "{{ cluster_state.infra_id }}" + +# Vars used on Bootstrap +bootstrap_bucket: "{{ _cluster_prefix }}-infra" + +# Vars used on Machine/Compute Stack +_instance_type: "{{ bootstrap_instance | d('m6i.xlarge') }}" +_instance_profile: "{{ cluster_state.compute.iam_profile_bootstrap }}" +# _image_id: "{{ custom_image_id | d(cluster_state.compute.image_id) }}" +_image_id: "rhcos-412.86.202212081411-0-openstack.x86_64" +# _subnet_name: "{{ _cluster_prefix }}-net-public-1a" + +_userdata_template: ocp-nodes-user-data.j2 +openshift_userdata: + config_source: "https://api-int.{{ cluster_state.dns.cluster_domain }}:22623/config/master" + ca_source: "{{ cluster_state.certificates.root_ca }}" + +## Common vars used in the Stack vars +# _common: +# prefix: "{{ _cluster_prefix }}-bootstrap" +# detailed_monitoring: yes +# ebs_optimized: no +# image_id: "{{ _image_id }}" +# instance_role: "{{ _instance_profile }}" +# instance_type: "{{ _instance_type }}" +# security_groups: +# - "{{ _cluster_prefix }}-bootstrap-sg" +# - "{{ _cluster_prefix }}-controlplane-sg" +# state: present +# tags: "{{ cluster_state.tags }}" +# termination_protection: no +# volumes: +# - device_name: /dev/xvda +# ebs: +# volume_size: 128 +# volume_type: gp3 +# delete_on_termination: true +# - device_name: /dev/xvdd +# ebs: +# volume_size: 32 +# volume_type: gp3 +# delete_on_termination: true + +# vpc_subnet_name: "{{ _subnet_name }}" +# wait: yes +# wait_timeout: 500 + +# Stack Compute (Ansible Role cloud_compute) options: +compute_resources: + # - provider: oci + # type: machine + # # name: "{{ cluster_state.infra_id }}-bootstrap{{ _machine_suffix }}" + + # # cloud_compute role to discovery for OCI: + # image_name: rhcos-412.86.202212081411-0-openstack.x86_64 + # vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public-nodes" + # network_security_group_names: + # - "{{ cluster_state.infra_id }}-nsg-bootstrap" + # # OCI spec + # spec: + # state: present + # compartment_id: "{{ oci_compartment_id }}" + # display_name: "{{ cluster_state.infra_id }}-master-03" + # region: "{{ config_cluster_region }}" + # #freeform_tags: {'Department': 'Finance'} + # #defined_tags: {'Operations': {'CostCenter': 'US'}} + # availability_domain: "gzqB:US-SANJOSE-1-AD-1" + # # platform_config: + # # type: AMD_VM + # shape: "VM.Standard.E4.Flex" + # shape_config: + # ocpus: 4 + # memory_in_gbs: 16 + # #baseline_ocpu_utilization: BASELINE_1_8 + # #nvmes: 1 + # fault_domain: FAULT-DOMAIN-1 + # # availability_domain: Uocm:PHX-AD-1 + # agent_config: + # are_all_plugins_disabled: true + + # source_details: + # source_type: image + + # create_vnic_details: + # display_name: "{{ cluster_state.infra_id }}-master-03-vnic0" + # assign_public_ip: true + # assign_private_dns_record: true + # hostname_label: "master-03" + # # defined_tags: {'Operations': {'CostCenter': 'US'}} + # # freeform_tags: {'Department': 'Finance'} + # # private_ip: private_ip_example + # # skip_source_dest_check: true + # # vlan_id: "ocid1.vlan.oc1..xxxxxxEXAMPLExxxxxx" + # #subnet_id: "{{ machine_subnet_id }}" + # #nsg_ids: "{{ machine_nsg_ids }}" + # metadata: + # user_data: "{{ lookup('template', _userdata_template) | to_nice_json | string | b64encode }}" + + # callbacks: + # - name: nlb + # # nlb_name: ocp-nlb + # nlb_name: "{{ cluster_state.infra_id }}-nlb" + # backend_sets: + # - name: "{{ cluster_state.infra_id }}-api" + # port: 6443 + # - name: "{{ cluster_state.infra_id }}-mcs" + # port: 22623 + # # - name: "6443" + # # port: 6443 + # # - name: "22623" + # # port: 22623 + + + - provider: oci + type: machine + + # cloud_compute role to discovery for OCI: + image_name: rhcos-412.86.202212081411-0-openstack.x86_64 + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public-nodes" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-bootstrap" + + # OCI spec + spec: + state: present + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-01" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-1 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-01-vnic0" + assign_public_ip: true + assign_private_dns_record: true + hostname_label: "master-01" + metadata: + user_data: "{{ lookup('template', _userdata_template) | to_nice_json | string | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + + + - provider: oci + type: machine + + # cloud_compute role to discovery for OCI: + image_name: rhcos-412.86.202212081411-0-openstack.x86_64 + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public-nodes" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-bootstrap" + + # OCI spec + spec: + state: present + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-02" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-2 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-02-vnic0" + assign_public_ip: true + assign_private_dns_record: true + hostname_label: "master-02" + metadata: + user_data: "{{ lookup('template', _userdata_template) | to_nice_json | string | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 + + + - provider: oci + type: machine + + # cloud_compute role to discovery for OCI: + image_name: rhcos-412.86.202212081411-0-openstack.x86_64 + vnic_subnet_name: "{{ cluster_state.infra_id }}-net-public-nodes" + network_security_group_names: + - "{{ cluster_state.infra_id }}-nsg-bootstrap" + + # OCI spec + spec: + state: present + compartment_id: "{{ oci_compartment_id }}" + display_name: "{{ cluster_state.infra_id }}-master-03" + region: "{{ config_cluster_region }}" + #freeform_tags: {'Department': 'Finance'} + #defined_tags: {'Operations': {'CostCenter': 'US'}} + availability_domain: "gzqB:US-SANJOSE-1-AD-1" + fault_domain: FAULT-DOMAIN-3 + + # platform_config: + # type: AMD_VM + shape: "VM.Standard.E4.Flex" + shape_config: + ocpus: 4 + memory_in_gbs: 16 + #baseline_ocpu_utilization: BASELINE_1_8 + #nvmes: 1 + agent_config: + are_all_plugins_disabled: true + + source_details: + source_type: image + + create_vnic_details: + display_name: "{{ cluster_state.infra_id }}-master-03-vnic0" + assign_public_ip: true + assign_private_dns_record: true + hostname_label: "master-03" + metadata: + user_data: "{{ lookup('template', _userdata_template) | to_nice_json | string | b64encode }}" + + callbacks: + - name: nlb + nlb_name: "{{ cluster_state.infra_id }}-nlb" + backend_sets: + - name: "{{ cluster_state.infra_id }}-api" + port: 6443 + - name: "{{ cluster_state.infra_id }}-mcs" + port: 22623 diff --git a/roles/bootstrap/tasks/oci.yaml b/roles/bootstrap/tasks/oci.yaml new file mode 100644 index 0000000..afbd2b5 --- /dev/null +++ b/roles/bootstrap/tasks/oci.yaml @@ -0,0 +1,57 @@ +--- +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_object_storage_object_module.html#ansible-collections-oracle-oci-oci-object-storage-object-module +# https://docs.oracle.com/en-us/iaas/tools/oci-ansible-collection/4.12.0/collections/oracle/oci/oci_object_storage_object_module.html#ansible-collections-oracle-oci-oci-object-storage-object-module + +- name: Get a specific namespace + oracle.oci.oci_object_storage_namespace_facts: + compartment_id: "{{ oci_compartment_id }}" + register: _objns + +- debug: var=_objns + +- name: Create bucket + oracle.oci.oci_object_storage_bucket: + compartment_id: "{{ oci_compartment_id }}" + name: "{{ bootstrap_bucket }}" + namespace_name: "{{ _objns.namespace }}" + state: present + +# TODO: Make it indepotent +- name: Upload bootstrap.ign + oracle.oci.oci_object_storage_object: + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + object_name: "/bootstrap.ign" + src: "{{ config_install_dir + '/' + bootstrap_src_ign }}" + force: false + register: _upload + +- name: OCI | Create expieration timestamp + ansible.builtin.command: "date +'%Y-%m-%dT%H:%M:%S%z' -d '+1 hour'" + register: _cmd + changed_when: false + +- name: OCI | Create preauthenticated_request + oracle.oci.oci_object_storage_preauthenticated_request: + name: par-bootstrap + access_type: ObjectRead + time_expires: "{{ _cmd.stdout }}" + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + object_name: "/bootstrap.ign" + register: _objpreauth + #when: _upload.changed + +- debug: var=_objpreauth + +- name: OCI | Show existing URLs + oracle.oci.oci_object_storage_preauthenticated_request_facts: + namespace_name: "{{ _objns.namespace }}" + bucket_name: "{{ bootstrap_bucket }}" + register: _pars + +- debug: var=_pars + +- name: OCI | Create Signed URL to bootstrap_bucket_signed_url + ansible.builtin.set_fact: + bootstrap_bucket_signed_url: "https://objectstorage.{{ config_cluster_region }}.oraclecloud.com{{ _objpreauth.preauthenticated_request.access_uri }}" diff --git a/roles/cloud_compute b/roles/cloud_compute index 5a0ea8a..cd201c8 160000 --- a/roles/cloud_compute +++ b/roles/cloud_compute @@ -1 +1 @@ -Subproject commit 5a0ea8a31f4ac8fe82e628fd6af8ef492ccee451 +Subproject commit cd201c8be746dfee6a4fb617bb874695fafb7c83 diff --git a/roles/cloud_load_balancer b/roles/cloud_load_balancer index 25ff915..7efc5b0 160000 --- a/roles/cloud_load_balancer +++ b/roles/cloud_load_balancer @@ -1 +1 @@ -Subproject commit 25ff915b0a72db6f0a1e86b6bc645a259f8348c1 +Subproject commit 7efc5b0dcbfd00bd2bdfa90486e9e6347eb54a9b diff --git a/roles/cloud_network b/roles/cloud_network index fabfc0e..2185a45 160000 --- a/roles/cloud_network +++ b/roles/cloud_network @@ -1 +1 @@ -Subproject commit fabfc0e0b9ec9e4622515c921b4c03642d6d8107 +Subproject commit 2185a45f91161ec01b9cbb213ad0f7b853a0ff33