From 84d5616c47c5609765509313b9129760fd195e19 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 9 Feb 2021 09:15:23 -0800 Subject: [PATCH] test: expose custom image options as env vars (#4244) --- test/e2e/cluster.sh | 5 + test/e2e/engine/template.go | 120 ++++++++++-------- .../availabilityset-standard-lb.json | 39 ------ .../test_cluster_configs/availabilityset.json | 7 - 4 files changed, 75 insertions(+), 96 deletions(-) delete mode 100644 test/e2e/test_cluster_configs/availabilityset-standard-lb.json diff --git a/test/e2e/cluster.sh b/test/e2e/cluster.sh index efc8f7c3a2..fd3a5d916f 100755 --- a/test/e2e/cluster.sh +++ b/test/e2e/cluster.sh @@ -239,6 +239,11 @@ docker run --rm \ -e KAMINO_VMSS_PROTOTYPE_IMAGE_REGISTRY=${KAMINO_VMSS_PROTOTYPE_IMAGE_REGISTRY} \ -e KAMINO_VMSS_PROTOTYPE_IMAGE_REPOSITORY=${KAMINO_VMSS_PROTOTYPE_IMAGE_REPOSITORY} \ -e KAMINO_VMSS_PROTOTYPE_IMAGE_TAG=${KAMINO_VMSS_PROTOTYPE_IMAGE_TAG} \ +-e CUSTOM_KUBE_PROXY_IMAGE=${CUSTOM_KUBE_PROXY_IMAGE} \ +-e CUSTOM_KUBE_APISERVER_IMAGE=${CUSTOM_KUBE_APISERVER_IMAGE} \ +-e CUSTOM_KUBE_SCHEDULER_IMAGE=${CUSTOM_KUBE_SCHEDULER_IMAGE} \ +-e CUSTOM_KUBE_CONTROLLER_MANAGER_IMAGE=${CUSTOM_KUBE_CONTROLLER_MANAGER_IMAGE} \ +-e CUSTOM_WINDOWS_PACKAGE_URL=${CUSTOM_WINDOWS_PACKAGE_URL} \ "${DEV_IMAGE}" make test-kubernetes || tryExit && renameResultsFile "deploy" if [ "${UPGRADE_CLUSTER}" = "true" ] || [ "${SCALE_CLUSTER}" = "true" ] || [ -n "$ADD_NODE_POOL_INPUT" ] || [ "${GET_CLUSTER_LOGS}" = "true" ] || [ "${ROTATE_CERTS}" = "true" ]; then diff --git a/test/e2e/engine/template.go b/test/e2e/engine/template.go index 8c43678eec..5ca7234b0b 100644 --- a/test/e2e/engine/template.go +++ b/test/e2e/engine/template.go @@ -28,56 +28,60 @@ import ( // Config represents the configuration values of a template stored as env vars type Config struct { - ClientID string `envconfig:"CLIENT_ID" required:"true"` - ClientSecret string `envconfig:"CLIENT_SECRET" required:"true"` - ClientObjectID string `envconfig:"CLIENT_OBJECTID" default:""` - LogAnalyticsWorkspaceKey string `envconfig:"LOG_ANALYTICS_WORKSPACE_KEY" default:""` - MasterDNSPrefix string `envconfig:"DNS_PREFIX" default:""` - AgentDNSPrefix string `envconfig:"DNS_PREFIX" default:""` - MSIUserAssignedID string `envconfig:"MSI_USER_ASSIGNED_ID" default:""` - UseManagedIdentity bool `envconfig:"USE_MANAGED_IDENTITY" default:"true"` - PublicSSHKey string `envconfig:"PUBLIC_SSH_KEY" default:""` - WindowsAdminPasssword string `envconfig:"WINDOWS_ADMIN_PASSWORD" default:""` - WindowsNodeImageGallery string `envconfig:"WINDOWS_NODE_IMAGE_GALLERY" default:""` - WindowsNodeImageName string `envconfig:"WINDOWS_NODE_IMAGE_NAME" default:""` - WindowsNodeImageResourceGroup string `envconfig:"WINDOWS_NODE_IMAGE_RESOURCE_GROUP" default:""` - WindowsNodeImageSubscriptionID string `envconfig:"WINDOWS_NODE_IMAGE_SUBSCRIPTION_ID" default:""` - WindowsNodeImageVersion string `envconfig:"WINDOWS_NODE_IMAGE_VERSION" default:""` - WindowsNodeVhdURL string `envconfig:"WINDOWS_NODE_VHD_URL" default:""` - LinuxNodeImageGallery string `envconfig:"LINUX_NODE_IMAGE_GALLERY" default:""` - LinuxNodeImageName string `envconfig:"LINUX_NODE_IMAGE_NAME" default:""` - LinuxNodeImageResourceGroup string `envconfig:"LINUX_NODE_IMAGE_RESOURCE_GROUP" default:""` - LinuxNodeImageSubscriptionID string `envconfig:"LINUX_NODE_IMAGE_SUBSCRIPTION_ID" default:""` - LinuxNodeImageVersion string `envconfig:"LINUX_NODE_IMAGE_VERSION" default:""` - OSDiskSizeGB string `envconfig:"OS_DISK_SIZE_GB" default:""` - ContainerRuntime string `envconfig:"CONTAINER_RUNTIME" default:""` - OrchestratorRelease string `envconfig:"ORCHESTRATOR_RELEASE" default:""` - OrchestratorVersion string `envconfig:"ORCHESTRATOR_VERSION" default:""` - OutputDirectory string `envconfig:"OUTPUT_DIR" default:"_output"` - CreateVNET bool `envconfig:"CREATE_VNET" default:"false"` - EnableKMSEncryption bool `envconfig:"ENABLE_KMS_ENCRYPTION" default:"false"` - Distro string `envconfig:"DISTRO" default:""` - SubscriptionID string `envconfig:"SUBSCRIPTION_ID" required:"true"` - InfraResourceGroup string `envconfig:"INFRA_RESOURCE_GROUP" default:""` - Location string `envconfig:"LOCATION" default:""` - TenantID string `envconfig:"TENANT_ID" required:"true"` - ImageName string `envconfig:"IMAGE_NAME" default:""` - ImageResourceGroup string `envconfig:"IMAGE_RESOURCE_GROUP" default:""` - DebugCrashingPods bool `envconfig:"DEBUG_CRASHING_PODS" default:"false"` - CustomHyperKubeImage string `envconfig:"CUSTOM_HYPERKUBE_IMAGE" default:""` - CustomKubeProxyImage string `envconfig:"CUSTOM_KUBE_PROXY_IMAGE" default:""` - EnableTelemetry bool `envconfig:"ENABLE_TELEMETRY" default:"true"` - KubernetesImageBase string `envconfig:"KUBERNETES_IMAGE_BASE" default:""` - KubernetesImageBaseType string `envconfig:"KUBERNETES_IMAGE_BASE_TYPE" default:""` - LinuxContainerdURL string `envconfig:"LINUX_CONTAINERD_URL"` - WindowsContainerdURL string `envconfig:"WINDOWS_CONTAINERD_URL"` - LinuxMobyURL string `envconfig:"LINUX_MOBY_URL"` - WindowsProvisioningScriptsURL string `envconfig:"WINDOWS_PROVISIONING_SCRIPTS_URL" default:""` - ArcClientID string `envconfig:"ARC_CLIENT_ID" default:""` - ArcClientSecret string `envconfig:"ARC_CLIENT_SECRET" default:""` - ArcSubscriptionID string `envconfig:"ARC_SUBSCRIPTION_ID" default:""` - ArcLocation string `envconfig:"ARC_LOCATION" default:""` - ArcTenantID string `envconfig:"ARC_TENANT_ID" default:""` + ClientID string `envconfig:"CLIENT_ID" required:"true"` + ClientSecret string `envconfig:"CLIENT_SECRET" required:"true"` + ClientObjectID string `envconfig:"CLIENT_OBJECTID" default:""` + LogAnalyticsWorkspaceKey string `envconfig:"LOG_ANALYTICS_WORKSPACE_KEY" default:""` + MasterDNSPrefix string `envconfig:"DNS_PREFIX" default:""` + AgentDNSPrefix string `envconfig:"DNS_PREFIX" default:""` + MSIUserAssignedID string `envconfig:"MSI_USER_ASSIGNED_ID" default:""` + UseManagedIdentity bool `envconfig:"USE_MANAGED_IDENTITY" default:"true"` + PublicSSHKey string `envconfig:"PUBLIC_SSH_KEY" default:""` + WindowsAdminPasssword string `envconfig:"WINDOWS_ADMIN_PASSWORD" default:""` + WindowsNodeImageGallery string `envconfig:"WINDOWS_NODE_IMAGE_GALLERY" default:""` + WindowsNodeImageName string `envconfig:"WINDOWS_NODE_IMAGE_NAME" default:""` + WindowsNodeImageResourceGroup string `envconfig:"WINDOWS_NODE_IMAGE_RESOURCE_GROUP" default:""` + WindowsNodeImageSubscriptionID string `envconfig:"WINDOWS_NODE_IMAGE_SUBSCRIPTION_ID" default:""` + WindowsNodeImageVersion string `envconfig:"WINDOWS_NODE_IMAGE_VERSION" default:""` + WindowsNodeVhdURL string `envconfig:"WINDOWS_NODE_VHD_URL" default:""` + LinuxNodeImageGallery string `envconfig:"LINUX_NODE_IMAGE_GALLERY" default:""` + LinuxNodeImageName string `envconfig:"LINUX_NODE_IMAGE_NAME" default:""` + LinuxNodeImageResourceGroup string `envconfig:"LINUX_NODE_IMAGE_RESOURCE_GROUP" default:""` + LinuxNodeImageSubscriptionID string `envconfig:"LINUX_NODE_IMAGE_SUBSCRIPTION_ID" default:""` + LinuxNodeImageVersion string `envconfig:"LINUX_NODE_IMAGE_VERSION" default:""` + OSDiskSizeGB string `envconfig:"OS_DISK_SIZE_GB" default:""` + ContainerRuntime string `envconfig:"CONTAINER_RUNTIME" default:""` + OrchestratorRelease string `envconfig:"ORCHESTRATOR_RELEASE" default:""` + OrchestratorVersion string `envconfig:"ORCHESTRATOR_VERSION" default:""` + OutputDirectory string `envconfig:"OUTPUT_DIR" default:"_output"` + CreateVNET bool `envconfig:"CREATE_VNET" default:"false"` + EnableKMSEncryption bool `envconfig:"ENABLE_KMS_ENCRYPTION" default:"false"` + Distro string `envconfig:"DISTRO" default:""` + SubscriptionID string `envconfig:"SUBSCRIPTION_ID" required:"true"` + InfraResourceGroup string `envconfig:"INFRA_RESOURCE_GROUP" default:""` + Location string `envconfig:"LOCATION" default:""` + TenantID string `envconfig:"TENANT_ID" required:"true"` + ImageName string `envconfig:"IMAGE_NAME" default:""` + ImageResourceGroup string `envconfig:"IMAGE_RESOURCE_GROUP" default:""` + DebugCrashingPods bool `envconfig:"DEBUG_CRASHING_PODS" default:"false"` + CustomHyperKubeImage string `envconfig:"CUSTOM_HYPERKUBE_IMAGE" default:""` + CustomKubeProxyImage string `envconfig:"CUSTOM_KUBE_PROXY_IMAGE" default:""` + CustomKubeAPIServerImage string `envconfig:"CUSTOM_KUBE_APISERVER_IMAGE" default:""` + CustomKubeSchedulerImage string `envconfig:"CUSTOM_KUBE_SCHEDULER_IMAGE" default:""` + CustomKubeControllerManagerImage string `envconfig:"CUSTOM_KUBE_CONTROLLER_MANAGER_IMAGE" default:""` + CustomWindowsPackageURL string `envconfig:"CUSTOM_WINDOWS_PACKAGE_URL" default:""` + EnableTelemetry bool `envconfig:"ENABLE_TELEMETRY" default:"true"` + KubernetesImageBase string `envconfig:"KUBERNETES_IMAGE_BASE" default:""` + KubernetesImageBaseType string `envconfig:"KUBERNETES_IMAGE_BASE_TYPE" default:""` + LinuxContainerdURL string `envconfig:"LINUX_CONTAINERD_URL"` + WindowsContainerdURL string `envconfig:"WINDOWS_CONTAINERD_URL"` + LinuxMobyURL string `envconfig:"LINUX_MOBY_URL"` + WindowsProvisioningScriptsURL string `envconfig:"WINDOWS_PROVISIONING_SCRIPTS_URL" default:""` + ArcClientID string `envconfig:"ARC_CLIENT_ID" default:""` + ArcClientSecret string `envconfig:"ARC_CLIENT_SECRET" default:""` + ArcSubscriptionID string `envconfig:"ARC_SUBSCRIPTION_ID" default:""` + ArcLocation string `envconfig:"ARC_LOCATION" default:""` + ArcTenantID string `envconfig:"ARC_TENANT_ID" default:""` ClusterDefinitionPath string // The original template we want to use to build the cluster from. ClusterDefinitionTemplate string // This is the template after we splice in the environment variables @@ -390,6 +394,22 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetIDs []string, i prop.OrchestratorProfile.KubernetesConfig.CustomKubeProxyImage = config.CustomKubeProxyImage } + if config.CustomKubeAPIServerImage != "" { + prop.OrchestratorProfile.KubernetesConfig.CustomKubeAPIServerImage = config.CustomKubeAPIServerImage + } + + if config.CustomKubeSchedulerImage != "" { + prop.OrchestratorProfile.KubernetesConfig.CustomKubeSchedulerImage = config.CustomKubeSchedulerImage + } + + if config.CustomKubeControllerManagerImage != "" { + prop.OrchestratorProfile.KubernetesConfig.CustomKubeControllerManagerImage = config.CustomKubeControllerManagerImage + } + + if config.CustomWindowsPackageURL != "" { + prop.OrchestratorProfile.KubernetesConfig.CustomWindowsPackageURL = config.CustomWindowsPackageURL + } + if config.EnableTelemetry == true { if prop.FeatureFlags == nil { prop.FeatureFlags = new(vlabs.FeatureFlags) diff --git a/test/e2e/test_cluster_configs/availabilityset-standard-lb.json b/test/e2e/test_cluster_configs/availabilityset-standard-lb.json deleted file mode 100644 index a76b014bc9..0000000000 --- a/test/e2e/test_cluster_configs/availabilityset-standard-lb.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "env": {}, - "apiModel": { - "apiVersion": "vlabs", - "properties": { - "orchestratorProfile": { - "kubernetesConfig": { - "loadBalancerSku": "Standard", - "excludeMasterFromStandardLB": true - } - }, - "masterProfile": { - "count": 1, - "dnsPrefix": "", - "vmSize": "Standard_D2_v3", - "OSDiskSizeGB": 200 - }, - "agentPoolProfiles": [ - { - "name": "agent1", - "count": 2, - "vmSize": "Standard_D2_v3", - "OSDiskSizeGB": 200, - "availabilityProfile": "AvailabilitySet" - } - ], - "linuxProfile": { - "adminUsername": "azureuser", - "ssh": { - "publicKeys": [ - { - "keyData": "" - } - ] - } - } - } - } -} diff --git a/test/e2e/test_cluster_configs/availabilityset.json b/test/e2e/test_cluster_configs/availabilityset.json index 131947e5bc..110c2a7d06 100644 --- a/test/e2e/test_cluster_configs/availabilityset.json +++ b/test/e2e/test_cluster_configs/availabilityset.json @@ -5,13 +5,6 @@ "STABILITY_ITERATIONS": "0", "GINKGO_SKIP_AFTER_UPGRADE": "dns-liveness pod" }, - "options": { - "allowedOrchestratorVersions": [ - "1.18", - "1.19", - "1.20" - ] - }, "apiModel": { "apiVersion": "vlabs", "properties": {