diff --git a/docs/topics/clusterdefinitions.md b/docs/topics/clusterdefinitions.md index aed881dcd8..747a7ede8f 100644 --- a/docs/topics/clusterdefinitions.md +++ b/docs/topics/clusterdefinitions.md @@ -52,7 +52,8 @@ $ aks-engine get-versions | WindowsSdnPluginURL | no (for development only) | **Experimental** - see [Windows ContainerD](features.md#windows-containerd) | | dnsServiceIP | no | IP address for coredns or kube-dns to listen on. If specified must be in the range of `serviceCidr` | | mobyVersion | no (for development only) | Enables an explicit moby version, e.g. `3.0.3`. Default is `3.0.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `mobyVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of moby. | -| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. This value is currently ignored for Windows. | +| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. This value is currently ignored for Windows. It is also ignored when `linuxContainerdURL` is defined. | +| linuxContainerdURL | no (for development only) | Allow the use of custom containerd package for Linux nodes. | | dockerBridgeSubnet | no | The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0) | | enableAggregatedAPIs | no | Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/). enableRbac must be set to true to use aggregated APIs. Aggregated API functionality is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default is true) | | enableDataEncryptionAtRest | no | Enable [kubernetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) | diff --git a/parts/k8s/cloud-init/artifacts/cse_main.sh b/parts/k8s/cloud-init/artifacts/cse_main.sh index 93fe18ade5..9a7e5c97a6 100755 --- a/parts/k8s/cloud-init/artifacts/cse_main.sh +++ b/parts/k8s/cloud-init/artifacts/cse_main.sh @@ -112,6 +112,14 @@ time_metric "InstallContainerd" installContainerd {{else}} time_metric "installMoby" installMoby {{end}} +{{- if HasLinuxContainerdURL}} + LINUX_CONTAINERD_URL={{GetLinuxContainerdURL}} + if [[ -n "${LINUX_CONTAINERD_URL:-}" ]]; then + DEB="${LINUX_CONTAINERD_URL##*/}" + retrycmd_no_stats 120 5 25 curl -fsSL ${LINUX_CONTAINERD_URL} >/tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_DOWNLOAD_TIMEOUT"}} + retrycmd 20 30 120 dpkg -i /tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_PKG_ADD_FAIL"}} + fi +{{end}} fi if [[ -n ${MASTER_NODE} ]] && [[ -z ${COSMOS_URI} ]]; then diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 8fbeae12ce..225ca20061 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -294,6 +294,7 @@ func convertKubernetesConfigToVLabs(apiCfg *KubernetesConfig, vlabsCfg *vlabs.Ku vlabsCfg.DockerBridgeSubnet = apiCfg.DockerBridgeSubnet vlabsCfg.MobyVersion = apiCfg.MobyVersion vlabsCfg.ContainerdVersion = apiCfg.ContainerdVersion + vlabsCfg.LinuxContainerdURL = apiCfg.LinuxContainerdURL vlabsCfg.CloudProviderBackoff = apiCfg.CloudProviderBackoff vlabsCfg.CloudProviderBackoffMode = apiCfg.CloudProviderBackoffMode vlabsCfg.CloudProviderBackoffDuration = apiCfg.CloudProviderBackoffDuration diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 050216ec1a..892d7418f7 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -310,6 +310,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.DockerBridgeSubnet = vlabs.DockerBridgeSubnet api.MobyVersion = vlabs.MobyVersion api.ContainerdVersion = vlabs.ContainerdVersion + api.LinuxContainerdURL = vlabs.LinuxContainerdURL api.CloudProviderBackoff = vlabs.CloudProviderBackoff api.CloudProviderBackoffMode = vlabs.CloudProviderBackoffMode api.CloudProviderBackoffDuration = vlabs.CloudProviderBackoffDuration diff --git a/pkg/api/types.go b/pkg/api/types.go index 7044e58862..34e29735e7 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -468,6 +468,7 @@ type KubernetesConfig struct { DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated MobyVersion string `json:"mobyVersion,omitempty"` ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 4f4df35014..73f8a390ee 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -353,6 +353,7 @@ type KubernetesConfig struct { DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated MobyVersion string `json:"mobyVersion,omitempty"` ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"` UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` diff --git a/pkg/engine/cse.go b/pkg/engine/cse.go index 786d5c89b4..2b6e6004b8 100644 --- a/pkg/engine/cse.go +++ b/pkg/engine/cse.go @@ -78,6 +78,8 @@ var cseErrorCodes = map[string]int{ "ERR_KUBERESERVED_SLICE_SETUP_FAIL": 181, "ERR_KUBELET_SLICE_SETUP_FAIL": 182, "ERR_CRI_SLICE_SETUP_FAIL": 183, + "ERR_DEB_DOWNLOAD_TIMEOUT": 184, + "ERR_DEB_PKG_ADD_FAIL": 185, } func GetCSEErrorCode(errorType string) int { diff --git a/pkg/engine/template_generator.go b/pkg/engine/template_generator.go index ae685fdce8..61369054a9 100644 --- a/pkg/engine/template_generator.go +++ b/pkg/engine/template_generator.go @@ -728,6 +728,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap { "NeedsContainerd": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.NeedsContainerd() }, + "HasLinuxContainerdURL": func() bool { + return cs.Properties.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL != "" + }, + "GetLinuxContainerdURL": func() string { + return cs.Properties.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL + }, "IsDockerContainerRuntime": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime == api.Docker }, diff --git a/pkg/engine/template_generator_test.go b/pkg/engine/template_generator_test.go index 0f50e18422..c5b6fb53ed 100644 --- a/pkg/engine/template_generator_test.go +++ b/pkg/engine/template_generator_test.go @@ -112,6 +112,8 @@ func TestGetTemplateFuncMap(t *testing.T) { "GetBase64EncodedEnvironmentJSON", "GetIdentitySystem", "NeedsContainerd", + "HasLinuxContainerdURL", + "GetLinuxContainerdURL", // TODO validate that the remaining func strings in getTemplateFuncMap are thinly wrapped and unit tested } @@ -146,6 +148,18 @@ func TestGetTemplateFuncMap(t *testing.T) { if ret[0].Interface() != false { t.Fatalf("Got unexpected NeedsContainerd response") } + case "HasLinuxContainerdURL": + rargs := make([]reflect.Value, 0) + ret := v.Call(rargs) + if ret[0].Interface() != false { + t.Fatalf("Got unexpected NeedsContainerd response") + } + case "GetLinuxContainerdURL": + rargs := make([]reflect.Value, 0) + ret := v.Call(rargs) + if ret[0].Interface() != "" { + t.Fatalf("Got unexpected NeedsContainerd response") + } } } } diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index a2a9be4182..a967d9bcf6 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -19905,6 +19905,14 @@ time_metric "InstallContainerd" installContainerd {{else}} time_metric "installMoby" installMoby {{end}} +{{- if HasLinuxContainerdURL}} + LINUX_CONTAINERD_URL={{GetLinuxContainerdURL}} + if [[ -n "${LINUX_CONTAINERD_URL:-}" ]]; then + DEB="${LINUX_CONTAINERD_URL##*/}" + retrycmd_no_stats 120 5 25 curl -fsSL ${LINUX_CONTAINERD_URL} >/tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_DOWNLOAD_TIMEOUT"}} + retrycmd 20 30 120 dpkg -i /tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_PKG_ADD_FAIL"}} + fi +{{end}} fi if [[ -n ${MASTER_NODE} ]] && [[ -z ${COSMOS_URI} ]]; then diff --git a/test/e2e/cluster.sh b/test/e2e/cluster.sh index c41ef8d4a5..bba341e242 100755 --- a/test/e2e/cluster.sh +++ b/test/e2e/cluster.sh @@ -185,6 +185,8 @@ docker run --rm \ -e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \ -e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \ -e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \ +-e LINUX_CONTAINERD_URL=${LINUX_CONTAINERD_URL} \ +-e WINDOWS_CONTAINERD_URL=${WINDOWS_CONTAINERD_URL} \ "${DEV_IMAGE}" make test-kubernetes || tryExit && renameResultsFile "deploy" if [ "${UPGRADE_CLUSTER}" = "true" ] || [ "${SCALE_CLUSTER}" = "true" ] || [ -n "$ADD_NODE_POOL_INPUT" ] || [ "${GET_CLUSTER_LOGS}" = "true" ] || [ "${ROTATE_CERTS}" = "true" ]; then diff --git a/test/e2e/engine/template.go b/test/e2e/engine/template.go index 3c26218a6b..9143449748 100644 --- a/test/e2e/engine/template.go +++ b/test/e2e/engine/template.go @@ -68,6 +68,8 @@ type Config struct { EnableTelemetry bool `envconfig:"ENABLE_TELEMETRY" default:"true"` KubernetesImageBase string `envconfig:"KUBERNETES_IMAGE_BASE" default:""` KubernetesImageBaseType string `envconfig:"KUBERNETES_IMAGE_BASE_TYPE" default:""` + LinuxContainerdURL string `envconfig:"LINUX_CONTAINERD_URL"` + WindowsContainerdURL string `envconfig:"WINDOWS_CONTAINERD_URL"` *ArcOnboardingConfig ClusterDefinitionPath string // The original template we want to use to build the cluster from. @@ -366,6 +368,14 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetIDs []string, i prop.OrchestratorProfile.KubernetesConfig.UserAssignedID = config.MSIUserAssignedID } + if config.LinuxContainerdURL != "" { + prop.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL = config.LinuxContainerdURL + } + + if config.WindowsContainerdURL != "" { + prop.OrchestratorProfile.KubernetesConfig.WindowsContainerdURL = config.WindowsContainerdURL + } + return &Engine{ Config: config, ClusterDefinition: cs, diff --git a/test/e2e/test_cluster_configs/containerd.json b/test/e2e/test_cluster_configs/containerd.json index 585909aff7..c336225c14 100644 --- a/test/e2e/test_cluster_configs/containerd.json +++ b/test/e2e/test_cluster_configs/containerd.json @@ -8,6 +8,7 @@ "kubernetesConfig": { "networkPlugin": "azure", "containerRuntime": "containerd", + "linuxContainerdURL": "https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod/pool/main/m/moby-containerd/moby-containerd_1.3.7+azure-2_amd64.deb", "windowsContainerdURL": "https://github.com/containerd/containerd/releases/download/v1.4.0/containerd-1.4.0-windows-amd64.tar.gz" } },