diff --git a/cluster-provision/README.md b/cluster-provision/README.md index 7850976092..4d745fd5e7 100644 --- a/cluster-provision/README.md +++ b/cluster-provision/README.md @@ -15,7 +15,7 @@ ## Versions to use * `kubevirtci/cli`: `sha256:1dd015dea4f12e6dcb8e31be3eeb677fed96f290ef4a4892a33c43d666053536` -* `kubevirtci/gocli`: `sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b` +* `kubevirtci/gocli`: `sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d` * `kubevirtci/base`: `sha256:850ac2e2828610b5f35f004f2a8a1ab23609a4c7891c8a1b68cbb7eef5f5dda0` * `kubevirtci/centos:1905_01`: `sha256:4b292b646f382d986c75a2be8ec49119a03467fe26dccc3a0886eb9e6e38c911` * `kubevirtci/centos:2001_01`: `sha256:6f2548dcc23489d0c945aef516781ae2ea678424c3760d1dafa0a83d29411713` diff --git a/cluster-provision/gocli/cmd/okd/provision.go b/cluster-provision/gocli/cmd/okd/provision.go index c1abd03f5e..65dbd3e2c9 100644 --- a/cluster-provision/gocli/cmd/okd/provision.go +++ b/cluster-provision/gocli/cmd/okd/provision.go @@ -33,6 +33,7 @@ func NewProvisionCommand() *cobra.Command { } provision.Flags().Bool("skip-cnao", false, "skip installing cluster network addons operator") + provision.Flags().String("networking-type", "OpenShiftSDN", "networking type: OpenShiftSDN, OVNKubernetes") provision.Flags().String("dir-hacks", "", "directory with installer hack that should be copied to the container") provision.Flags().String("dir-manifests", "", "directory with additional manifests that should be installed") provision.Flags().String("dir-scripts", "", "directory with scripts that should be copied to the container") @@ -80,6 +81,12 @@ func provision(cmd *cobra.Command, args []string) error { } envs = append(envs, fmt.Sprintf("CNAO=%t", !skipCnao)) + networkingType, err := cmd.Flags().GetString("networking-type") + if err != nil { + return err + } + envs = append(envs, fmt.Sprintf("NETWORKING_TYPE=%s", networkingType)) + masterMemory, err := cmd.Flags().GetString("master-memory") if err != nil { return err diff --git a/cluster-provision/manifests/okd/install-config.yaml b/cluster-provision/manifests/okd/install-config.yaml index 7fdd5e966e..f71003b5c6 100644 --- a/cluster-provision/manifests/okd/install-config.yaml +++ b/cluster-provision/manifests/okd/install-config.yaml @@ -16,7 +16,7 @@ networking: hostSubnetLength: 9 machineCIDR: 192.168.126.0/24 serviceCIDR: 172.30.0.0/16 - type: OpenShiftSDN + type: ${NETWORKING_TYPE} platform: libvirt: URI: qemu+tcp://192.168.122.1/system diff --git a/cluster-provision/manifests/okd/local-storage.yaml b/cluster-provision/manifests/okd/local-storage.yaml index a12f9a8477..edb01f2269 100644 --- a/cluster-provision/manifests/okd/local-storage.yaml +++ b/cluster-provision/manifests/okd/local-storage.yaml @@ -1,5 +1,5 @@ --- -apiVersion: operators.coreos.com/v1alpha2 +apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: local-operator-group diff --git a/cluster-provision/ocp/4.3/provision.sh b/cluster-provision/ocp/4.3/provision.sh index e905f6eb07..03b44262ef 100755 --- a/cluster-provision/ocp/4.3/provision.sh +++ b/cluster-provision/ocp/4.3/provision.sh @@ -6,7 +6,7 @@ PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" KUBEVIRTCI_DIR="$( cd ${PARENT_DIR}/../kubevirtci && pwd)" okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run \ --privileged \ diff --git a/cluster-provision/ocp/4.4/provision.sh b/cluster-provision/ocp/4.4/provision.sh new file mode 100755 index 0000000000..33484c4319 --- /dev/null +++ b/cluster-provision/ocp/4.4/provision.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -x + +PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" +KUBEVIRTCI_DIR="$( cd ${PARENT_DIR}/../kubevirtci && pwd)" + +okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" + +gocli="docker run \ +--privileged \ +--net=host \ +--rm -t \ +-v /var/run/docker.sock:/var/run/docker.sock \ +-v ${PARENT_DIR}:${PARENT_DIR} \ +docker.io/kubevirtci/gocli@${gocli_image_hash}" + +provisioner_container_id=$(docker ps --filter name=ocp-4.4-provision-cluster --format {{.ID}}) +docker kill $provisioner_container_id +docker container rm $provisioner_container_id + +# For ocp-4.4 we want OVNKubernetes +${gocli} provision okd \ +--prefix ocp-4.4-provision \ +--dir-scripts ${PARENT_DIR}/okd/scripts \ +--dir-manifests ${PARENT_DIR}/manifests \ +--dir-hacks ${PARENT_DIR}/okd/hacks \ +--skip-cnao \ +--master-memory 10240 \ +--workers-memory 8192 \ +--workers-cpu 4 \ +--networking-type OVNKubernetes \ +--installer-pull-secret-file ${INSTALLER_PULL_SECRET} \ +--installer-repo-tag release-4.4 \ +--installer-release-image registry.svc.ci.openshift.org/ocp/release:4.4 \ +"kubevirtci/okd-base@${okd_base_hash}" +rc=$? + + + +exit $rc diff --git a/cluster-provision/ocp/4.4/publish.sh b/cluster-provision/ocp/4.4/publish.sh new file mode 100755 index 0000000000..00b875dec3 --- /dev/null +++ b/cluster-provision/ocp/4.4/publish.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +tag=$(git log -1 --pretty=%h)-$(date +%s) +destination="quay.io/kubevirtci/ocp-4.4:$tag" + +docker tag kubevirtci/ocp-4.4-provision:latest $destination +docker push $destination diff --git a/cluster-provision/ocp/4.4/run.sh b/cluster-provision/ocp/4.4/run.sh new file mode 100755 index 0000000000..a556246bc6 --- /dev/null +++ b/cluster-provision/ocp/4.4/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -x + +ocp_image_hash="sha256:16a70403141142aae387a50feb2fd039a745c6916aa3f61e1a5d5a74efb6be39" +gocli_image_hash="sha256:a7880757e2d2755c6a784c1b64c64b096769ed3ccfac9d8e535df481731c2144" + +gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" + +${gocli} run ocp --random-ports --background --prefix ocp-4.4 --registry-volume ocp-4.4-registry "kubevirtci/ocp-4.4@${ocp_image_hash}" diff --git a/cluster-provision/okd/4.1/provision.sh b/cluster-provision/okd/4.1/provision.sh index f668416386..09da131879 100755 --- a/cluster-provision/okd/4.1/provision.sh +++ b/cluster-provision/okd/4.1/provision.sh @@ -5,7 +5,7 @@ set -x PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run \ --privileged \ diff --git a/cluster-provision/okd/4.1/run.sh b/cluster-provision/okd/4.1/run.sh index 285fd73e67..a9af5f5961 100755 --- a/cluster-provision/okd/4.1/run.sh +++ b/cluster-provision/okd/4.1/run.sh @@ -3,7 +3,7 @@ set -x okd_image_hash="sha256:e7e3a03bb144eb8c0be4dcd700592934856fb623d51a2b53871d69267ca51c86" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" diff --git a/cluster-provision/okd/4.2/provision.sh b/cluster-provision/okd/4.2/provision.sh index 984887a030..3eae7ced54 100755 --- a/cluster-provision/okd/4.2/provision.sh +++ b/cluster-provision/okd/4.2/provision.sh @@ -5,7 +5,7 @@ set -x PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run \ --privileged \ diff --git a/cluster-provision/okd/4.2/run.sh b/cluster-provision/okd/4.2/run.sh index e1a2c5e6a6..fcbee924fb 100755 --- a/cluster-provision/okd/4.2/run.sh +++ b/cluster-provision/okd/4.2/run.sh @@ -3,7 +3,7 @@ set -x okd_image_hash="sha256:a830064ca7bf5c5c2f15df180f816534e669a9a038fef4919116d61eb33e84c5" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" diff --git a/cluster-provision/okd/4.3/provision.sh b/cluster-provision/okd/4.3/provision.sh index c862497449..d810c43bfe 100755 --- a/cluster-provision/okd/4.3/provision.sh +++ b/cluster-provision/okd/4.3/provision.sh @@ -5,7 +5,7 @@ set -x PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. && pwd )" okd_base_hash="sha256:73ede51ce464546a82b81956b7f58cf98662a4c5fded9c659b57746bc131e047" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run \ --privileged \ diff --git a/cluster-provision/okd/4.3/run.sh b/cluster-provision/okd/4.3/run.sh index c26595ccbd..7113097e94 100755 --- a/cluster-provision/okd/4.3/run.sh +++ b/cluster-provision/okd/4.3/run.sh @@ -3,7 +3,7 @@ set -x okd_image_hash="sha256:63abc3884002a615712dfac5f42785be864ea62006892bf8a086ccdbca8b3d38" -gocli_image_hash="sha256:8dc7a694e67fadfbb337d59dfc269253079e31dca62e5298361dd464a82adc4b" +gocli_image_hash="sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d" gocli="docker run --privileged --net=host --rm -t -v /var/run/docker.sock:/var/run/docker.sock docker.io/kubevirtci/gocli@${gocli_image_hash}" diff --git a/cluster-provision/okd/hacks/release-4.4 b/cluster-provision/okd/hacks/release-4.4 new file mode 100644 index 0000000000..3f101835db --- /dev/null +++ b/cluster-provision/okd/hacks/release-4.4 @@ -0,0 +1,210 @@ +diff --git a/cmd/openshift-install/create.go b/cmd/openshift-install/create.go +index f9ae4c6bb..dea45f0d7 100644 +--- a/cmd/openshift-install/create.go ++++ b/cmd/openshift-install/create.go +@@ -244,7 +244,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director + + discovery := client.Discovery() + +- apiTimeout := 30 * time.Minute ++ apiTimeout := 120 * time.Minute + logrus.Infof("Waiting up to %v for the Kubernetes API at %s...", apiTimeout, config.Host) + apiContext, cancel := context.WithTimeout(ctx, apiTimeout) + defer cancel() +@@ -285,7 +285,7 @@ func waitForBootstrapComplete(ctx context.Context, config *rest.Config, director + // and waits for the bootstrap configmap to report that bootstrapping has + // completed. + func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset) error { +- timeout := 30 * time.Minute ++ timeout := 120 * time.Minute + logrus.Infof("Waiting up to %v for bootstrapping to complete...", timeout) + + waitCtx, cancel := context.WithTimeout(ctx, timeout) +@@ -323,7 +323,7 @@ func waitForBootstrapConfigMap(ctx context.Context, client *kubernetes.Clientset + // waitForInitializedCluster watches the ClusterVersion waiting for confirmation + // that the cluster has been initialized. + func waitForInitializedCluster(ctx context.Context, config *rest.Config) error { +- timeout := 30 * time.Minute ++ timeout := 120 * time.Minute + logrus.Infof("Waiting up to %v for the cluster at %s to initialize...", timeout, config.Host) + cc, err := configclient.NewForConfig(config) + if err != nil { +diff --git a/data/data/libvirt/main.tf b/data/data/libvirt/main.tf +index 9ba88c9cf..09f6500bf 100644 +--- a/data/data/libvirt/main.tf ++++ b/data/data/libvirt/main.tf +@@ -33,6 +33,7 @@ resource "libvirt_volume" "master" { + name = "${var.cluster_id}-master-${count.index}" + base_volume_id = module.volume.coreos_base_volume_id + pool = libvirt_pool.storage_pool.name ++ size = 32212254720 + } + + resource "libvirt_ignition" "master" { +@@ -73,6 +74,8 @@ resource "libvirt_network" "net" { + data.libvirt_network_dns_host_template.masters.*.rendered, + data.libvirt_network_dns_host_template.masters_int.*.rendered, + data.libvirt_network_dns_host_template.etcds.*.rendered, ++ data.libvirt_network_dns_host_template.console.*.rendered, ++ data.libvirt_network_dns_host_template.auth.*.rendered, + ) + content { + hostname = hosts.value.hostname +@@ -114,6 +117,19 @@ resource "libvirt_domain" "master" { + } + } + ++data "libvirt_network_dns_host_template" "auth" { ++ count = "${var.master_count}" ++ ip = "${var.libvirt_auth_ip}" ++ hostname = "oauth-openshift.apps.${var.cluster_domain}" ++} ++ ++data "libvirt_network_dns_host_template" "console" { ++ count = "${var.master_count}" ++ ip = "${var.libvirt_auth_ip}" ++ hostname = "console-openshift-console.apps.${var.cluster_domain}" ++} ++ ++ + data "libvirt_network_dns_host_template" "bootstrap" { + count = var.bootstrap_dns ? 1 : 0 + ip = var.libvirt_bootstrap_ip +diff --git a/data/data/libvirt/variables-libvirt.tf b/data/data/libvirt/variables-libvirt.tf +index 53cf68bae..3c5f7f905 100644 +--- a/data/data/libvirt/variables-libvirt.tf ++++ b/data/data/libvirt/variables-libvirt.tf +@@ -28,6 +28,11 @@ variable "libvirt_master_ips" { + description = "the list of desired master ips. Must match master_count" + } + ++variable "libvirt_auth_ip" { ++ type = "string" ++ description = "node with authentication server ip" ++} ++ + # It's definitely recommended to bump this if you can. + variable "libvirt_master_memory" { + type = string +diff --git a/pkg/asset/tls/aggregator.go b/pkg/asset/tls/aggregator.go +index 9ec6432da..6dac0b736 100644 +--- a/pkg/asset/tls/aggregator.go ++++ b/pkg/asset/tls/aggregator.go +@@ -27,7 +27,7 @@ func (a *AggregatorCA) Generate(dependencies asset.Parents) error { + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "aggregator", OrganizationalUnit: []string{"bootkube"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + IsCA: true, + } + +@@ -65,7 +65,7 @@ func (a *APIServerProxyCertKey) Generate(dependencies asset.Parents) error { + Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + } + + return a.SignedCertKey.Generate(cfg, aggregatorCA, "apiserver-proxy", DoNotAppendParent) +@@ -93,7 +93,7 @@ func (c *AggregatorSignerCertKey) Generate(parents asset.Parents) error { + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "aggregator-signer", OrganizationalUnit: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + IsCA: true, + } + +@@ -158,7 +158,7 @@ func (a *AggregatorClientCertKey) Generate(dependencies asset.Parents) error { + Subject: pkix.Name{CommonName: "system:kube-apiserver-proxy", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + } + + return a.SignedCertKey.Generate(cfg, ca, "aggregator-client", DoNotAppendParent) +diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go +index a50bee836..cd63ff13c 100644 +--- a/pkg/asset/tls/apiserver.go ++++ b/pkg/asset/tls/apiserver.go +@@ -185,7 +185,7 @@ func (a *KubeAPIServerLocalhostServerCertKey) Generate(dependencies asset.Parent + Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + DNSNames: []string{ + "localhost", + }, +@@ -288,7 +288,7 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P + Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + DNSNames: []string{ + "kubernetes", "kubernetes.default", + "kubernetes.default.svc", +@@ -392,7 +392,7 @@ func (a *KubeAPIServerExternalLBServerCertKey) Generate(dependencies asset.Paren + Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + DNSNames: []string{ + apiAddress(installConfig.Config), + }, +@@ -431,7 +431,7 @@ func (a *KubeAPIServerInternalLBServerCertKey) Generate(dependencies asset.Paren + Subject: pkix.Name{CommonName: "system:kube-apiserver", Organization: []string{"kube-master"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + DNSNames: []string{ + internalAPIAddress(installConfig.Config), + }, +diff --git a/pkg/asset/tls/kubelet.go b/pkg/asset/tls/kubelet.go +index 01264e898..32cc8059d 100644 +--- a/pkg/asset/tls/kubelet.go ++++ b/pkg/asset/tls/kubelet.go +@@ -24,7 +24,7 @@ func (c *KubeletCSRSignerCertKey) Generate(parents asset.Parents) error { + cfg := &CertCfg{ + Subject: pkix.Name{CommonName: "kubelet-signer", OrganizationalUnit: []string{"openshift"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + IsCA: true, + } + +@@ -181,7 +181,7 @@ func (a *KubeletClientCertKey) Generate(dependencies asset.Parents) error { + Subject: pkix.Name{CommonName: "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper", Organization: []string{"system:serviceaccounts:openshift-machine-config-operator"}}, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, +- Validity: ValidityOneDay, ++ Validity: ValidityOneYear, + } + + return a.SignedCertKey.Generate(cfg, ca, "kubelet-client", DoNotAppendParent) +diff --git a/pkg/tfvars/libvirt/libvirt.go b/pkg/tfvars/libvirt/libvirt.go +index a51fbfba1..7542dc278 100644 +--- a/pkg/tfvars/libvirt/libvirt.go ++++ b/pkg/tfvars/libvirt/libvirt.go +@@ -20,6 +20,7 @@ type config struct { + BootstrapIP string `json:"libvirt_bootstrap_ip,omitempty"` + MasterMemory string `json:"libvirt_master_memory,omitempty"` + MasterVcpu string `json:"libvirt_master_vcpu,omitempty"` ++ AuthNodeIP string `json:"libvirt_auth_ip,omitempty"` + } + + // TFVars generates libvirt-specific Terraform variables. +@@ -45,6 +46,7 @@ func TFVars(masterConfig *v1beta1.LibvirtMachineProviderConfig, osImage string, + IfName: bridge, + BootstrapIP: bootstrapIP.String(), + MasterIPs: masterIPs, ++ AuthNodeIP: "192.168.126.51", + MasterMemory: strconv.Itoa(masterConfig.DomainMemory), + MasterVcpu: strconv.Itoa(masterConfig.DomainVcpu), + } diff --git a/cluster-provision/okd/scripts/provision.sh b/cluster-provision/okd/scripts/provision.sh index 173e9036da..f6c09835f6 100755 --- a/cluster-provision/okd/scripts/provision.sh +++ b/cluster-provision/okd/scripts/provision.sh @@ -122,6 +122,7 @@ envsubst < /manifests/okd/registries.yaml > /registries.yaml set +x export PULL_SECRET=$(cat /etc/installer/token) export SSH_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" + envsubst < /manifests/okd/install-config.yaml > ${INSTALL_CONFIG_FILE} unset PULL_SECRET set -x @@ -170,6 +171,9 @@ cp "${CLUSTER_DIR}/openshift/99-worker-registries.yaml" ./ # Generate ignition configs /openshift-install --dir "${CLUSTER_DIR}" create ignition-configs +# Clean up memory cache so we have all resources available +sync; echo 3 > /proc/sys/vm/drop_caches + # Excecute installer export TF_VAR_libvirt_master_memory=$MASTER_MEMORY export TF_VAR_libvirt_master_vcpu=$MASTER_CPU diff --git a/cluster-up/cluster/images.sh b/cluster-up/cluster/images.sh index f9432a37b4..295aaf18a7 100644 --- a/cluster-up/cluster/images.sh +++ b/cluster-up/cluster/images.sh @@ -17,6 +17,7 @@ if [ -z $KUBEVIRTCI_PROVISION_CHECK ]; then IMAGES[okd-4.2]="okd-4.2@sha256:a830064ca7bf5c5c2f15df180f816534e669a9a038fef4919116d61eb33e84c5" IMAGES[okd-4.3]="okd-4.3@sha256:63abc3884002a615712dfac5f42785be864ea62006892bf8a086ccdbca8b3d38" IMAGES[ocp-4.3]="ocp-4.3@sha256:03a8c736263493961f198b5cb214d9b1fc265ece233c60bdb1c8b8b4b779ee1e" + IMAGES[ocp-4.4]="ocp-4.4@sha256:b235e87323ed88c46fedf27e9115573b92f228a82559ab7523dd1be183f66af8" fi export IMAGES diff --git a/cluster-up/cluster/ocp-4.3/provider.sh b/cluster-up/cluster/ocp-4.3/provider.sh index cdf525e435..50ae4d4e4a 100755 --- a/cluster-up/cluster/ocp-4.3/provider.sh +++ b/cluster-up/cluster/ocp-4.3/provider.sh @@ -51,6 +51,9 @@ function up() { params=" --container-registry= $params" fi + # Free some cached/buffered mem + sync; echo 3 > /proc/sys/vm/drop_caches + ${_cli} run okd ${params} --container-registry-user $user --container-registry-password $password # Copy k8s config and kubectl diff --git a/cluster-up/cluster/ocp-4.4/README.md b/cluster-up/cluster/ocp-4.4/README.md new file mode 100644 index 0000000000..267a47c575 --- /dev/null +++ b/cluster-up/cluster/ocp-4.4/README.md @@ -0,0 +1,57 @@ +# OCP 4.4 in ephemeral containers + +Provides a pre-deployed OCP with version 4.4 purely in docker +containers with libvirt. The provided VMs are completely ephemeral and are +recreated on every cluster restart. The KubeVirt containers are built on the +local machine and are then pushed to a registry which is exposed at +`localhost:5000`. + +It also comes with OVNKubernetes at ocp networking type instead of OpenShiftSDN +you can follow guide to play with it [1]. + +[1] https://github.com/ovn-org/ovn-kubernetes/blob/master/README_MANUAL.md + +## Bringing the cluster up + +The container is stored at a private repository at quay.io/kubevirtci, you +have to ask for pull permissions there and do a docker login before cluster-up + +```bash +docker login -u [quay user] -p [quay password] quay.io +``` + +```bash +export KUBEVIRT_PROVIDER=ocp-4.4 +export KUBEVIRT_NUM_NODES=3 # master + two workers +make cluster-up +``` + +The cluster can be accessed as usual: + +```bash +$ cluster/kubectl.sh get nodes +NAME STATUS ROLES AGE VERSION +test-1-82xp6-master-0 Ready master 62m v1.12.4+509916ce1 +test-1-82xp6-worker-0-wxf27 Ready worker 57m v1.12.4+509916ce1 +``` + +## Bringing the cluster down + +```bash +export KUBEVIRT_PROVIDER=ocp-4.4 +make cluster-down +``` + +This destroys the whole cluster. Recreating the cluster is fast, since OCP is +already pre-deployed. The only state which is kept is the state of the local +docker registry. + +## Destroying the docker registry state + +The docker registry survives a `make cluster-down`. It's state is stored in a +docker volume called `kubevirt_registry`. If the volume gets too big or the +volume contains corrupt data, it can be deleted with + +```bash +docker volume rm kubevirt_registry +``` diff --git a/cluster-up/cluster/ocp-4.4/provider.sh b/cluster-up/cluster/ocp-4.4/provider.sh new file mode 100755 index 0000000000..50ae4d4e4a --- /dev/null +++ b/cluster-up/cluster/ocp-4.4/provider.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -e + +source ${KUBEVIRTCI_PATH}/cluster/ephemeral-provider-common.sh + +function _port() { + ${_cli} ports --prefix $provider_prefix --container-name cluster "$@" +} + +function _install_from_cluster() { + local src_cid="$1" + local src_file="$2" + local dst_perms="$3" + local dst_file="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/$4" + + touch $dst_file + chmod $dst_perms $dst_file + docker exec $src_cid cat $src_file > $dst_file +} + +function up() { + container_registry="quay.io" + workers=$(($KUBEVIRT_NUM_NODES-1)) + if [[ ( $workers < 1 ) ]]; then + workers=1 + fi + echo "Number of workers: $workers" + params="--random-ports --background --prefix $provider_prefix --master-cpu 6 --workers-cpu 6 --workers-memory 8192 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --registry-volume $(_registry_volume) --workers $workers kubevirtci/${image}" + if [[ ! -z "${RHEL_NFS_DIR}" ]]; then + params=" --nfs-data $RHEL_NFS_DIR ${params}" + fi + + if [[ ! -z "${OKD_CONSOLE_PORT}" ]]; then + params=" --ocp-console-port $OKD_CONSOLE_PORT ${params}" + fi + + if [[ ! -z "${INSTALLER_PULL_SECRET}" ]]; then + params=" --installer-pull-secret-file ${INSTALLER_PULL_SECRET} ${params}" + fi + + # The auth has the format base64(user:password) + auth=$(cat ~/.docker/config.json | docker run --rm -i imega/jq:1.6 -r '.auths["'$container_registry'"]["auth"]' |base64 -d) + user=$(echo $auth |awk -F: '{print $1}') + password=$(echo $auth |awk -F: '{print $2}') + + # If provision test mode is on, use local image + if [ -z $KUBEVIRTCI_PROVISION_CHECK ]; then + params=" --container-registry ${container_registry} $params" + else + params=" --container-registry= $params" + fi + + # Free some cached/buffered mem + sync; echo 3 > /proc/sys/vm/drop_caches + + ${_cli} run okd ${params} --container-registry-user $user --container-registry-password $password + + # Copy k8s config and kubectl + cluster_container_id=$(docker ps -f "name=$provider_prefix-cluster" --format "{{.ID}}") + + _install_from_cluster $cluster_container_id /usr/local/bin/oc 0755 .kubectl + _install_from_cluster $cluster_container_id /root/install/auth/kubeconfig 0644 .kubeconfig + + # Set server and disable tls check + export KUBECONFIG=${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubeconfig + ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster test-1 --server=https://$(_main_ip):$(_port k8s) + ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kubectl config set-cluster test-1 --insecure-skip-tls-verify=true + + # Make sure that local config is correct + prepare_config +}