Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

Commit

Permalink
feat: allow custom containerd package for Linux nodes (#3878)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ernest Wong authored Oct 2, 2020
1 parent 0074c24 commit 7dc01ab
Show file tree
Hide file tree
Showing 13 changed files with 57 additions and 1 deletion.
3 changes: 2 additions & 1 deletion docs/topics/clusterdefinitions.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ $ aks-engine get-versions
| WindowsSdnPluginURL | no (for development only) | **Experimental** - see [Windows ContainerD](features.md#windows-containerd) |
| dnsServiceIP | no | IP address for coredns or kube-dns to listen on. If specified must be in the range of `serviceCidr` |
| mobyVersion | no (for development only) | Enables an explicit moby version, e.g. `3.0.3`. Default is `3.0.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `mobyVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of moby. |
| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. This value is currently ignored for Windows. |
| containerdVersion | no (for development only) | Enables an explicit containerd version, e.g. `1.1.4`. Default is `1.1.5`. This `kubernetesConfig` property is for development only, and applies only to cluster creation: `aks-engine upgrade` will always statically set `containerdVersion` to the default version at the time of upgrade, to ensure that upgraded clusters have the most recent, validated version of containerd. This value is currently ignored for Windows. It is also ignored when `linuxContainerdURL` is defined. |
| linuxContainerdURL | no (for development only) | Allow the use of custom containerd package for Linux nodes. |
| dockerBridgeSubnet | no | The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0) |
| enableAggregatedAPIs | no | Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/). enableRbac must be set to true to use aggregated APIs. Aggregated API functionality is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default is true) |
| enableDataEncryptionAtRest | no | Enable [kubernetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) |
Expand Down
8 changes: 8 additions & 0 deletions parts/k8s/cloud-init/artifacts/cse_main.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,14 @@ time_metric "InstallContainerd" installContainerd
{{else}}
time_metric "installMoby" installMoby
{{end}}
{{- if HasLinuxContainerdURL}}
LINUX_CONTAINERD_URL={{GetLinuxContainerdURL}}
if [[ -n "${LINUX_CONTAINERD_URL:-}" ]]; then
DEB="${LINUX_CONTAINERD_URL##*/}"
retrycmd_no_stats 120 5 25 curl -fsSL ${LINUX_CONTAINERD_URL} >/tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_DOWNLOAD_TIMEOUT"}}
retrycmd 20 30 120 dpkg -i /tmp/${DEB} || exit {{GetCSEErrorCode "ERR_DEB_PKG_ADD_FAIL"}}
fi
{{end}}
fi

if [[ -n ${MASTER_NODE} ]] && [[ -z ${COSMOS_URI} ]]; then
Expand Down
1 change: 1 addition & 0 deletions pkg/api/converterfromapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,7 @@ func convertKubernetesConfigToVLabs(apiCfg *KubernetesConfig, vlabsCfg *vlabs.Ku
vlabsCfg.DockerBridgeSubnet = apiCfg.DockerBridgeSubnet
vlabsCfg.MobyVersion = apiCfg.MobyVersion
vlabsCfg.ContainerdVersion = apiCfg.ContainerdVersion
vlabsCfg.LinuxContainerdURL = apiCfg.LinuxContainerdURL
vlabsCfg.CloudProviderBackoff = apiCfg.CloudProviderBackoff
vlabsCfg.CloudProviderBackoffMode = apiCfg.CloudProviderBackoffMode
vlabsCfg.CloudProviderBackoffDuration = apiCfg.CloudProviderBackoffDuration
Expand Down
1 change: 1 addition & 0 deletions pkg/api/convertertoapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
api.DockerBridgeSubnet = vlabs.DockerBridgeSubnet
api.MobyVersion = vlabs.MobyVersion
api.ContainerdVersion = vlabs.ContainerdVersion
api.LinuxContainerdURL = vlabs.LinuxContainerdURL
api.CloudProviderBackoff = vlabs.CloudProviderBackoff
api.CloudProviderBackoffMode = vlabs.CloudProviderBackoffMode
api.CloudProviderBackoffDuration = vlabs.CloudProviderBackoffDuration
Expand Down
1 change: 1 addition & 0 deletions pkg/api/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ type KubernetesConfig struct {
DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated
MobyVersion string `json:"mobyVersion,omitempty"`
ContainerdVersion string `json:"containerdVersion,omitempty"`
LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
Expand Down
1 change: 1 addition & 0 deletions pkg/api/vlabs/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@ type KubernetesConfig struct {
DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated
MobyVersion string `json:"mobyVersion,omitempty"`
ContainerdVersion string `json:"containerdVersion,omitempty"`
LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"`
CustomCcmImage string `json:"customCcmImage,omitempty"`
UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"`
CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"`
Expand Down
2 changes: 2 additions & 0 deletions pkg/engine/cse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ var cseErrorCodes = map[string]int{
"ERR_KUBERESERVED_SLICE_SETUP_FAIL": 181,
"ERR_KUBELET_SLICE_SETUP_FAIL": 182,
"ERR_CRI_SLICE_SETUP_FAIL": 183,
"ERR_DEB_DOWNLOAD_TIMEOUT": 184,
"ERR_DEB_PKG_ADD_FAIL": 185,
}

func GetCSEErrorCode(errorType string) int {
Expand Down
6 changes: 6 additions & 0 deletions pkg/engine/template_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,12 @@ func getContainerServiceFuncMap(cs *api.ContainerService) template.FuncMap {
"NeedsContainerd": func() bool {
return cs.Properties.OrchestratorProfile.KubernetesConfig.NeedsContainerd()
},
"HasLinuxContainerdURL": func() bool {
return cs.Properties.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL != ""
},
"GetLinuxContainerdURL": func() string {
return cs.Properties.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL
},
"IsDockerContainerRuntime": func() bool {
return cs.Properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime == api.Docker
},
Expand Down
14 changes: 14 additions & 0 deletions pkg/engine/template_generator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ func TestGetTemplateFuncMap(t *testing.T) {
"GetBase64EncodedEnvironmentJSON",
"GetIdentitySystem",
"NeedsContainerd",
"HasLinuxContainerdURL",
"GetLinuxContainerdURL",
// TODO validate that the remaining func strings in getTemplateFuncMap are thinly wrapped and unit tested
}

Expand Down Expand Up @@ -146,6 +148,18 @@ func TestGetTemplateFuncMap(t *testing.T) {
if ret[0].Interface() != false {
t.Fatalf("Got unexpected NeedsContainerd response")
}
case "HasLinuxContainerdURL":
rargs := make([]reflect.Value, 0)
ret := v.Call(rargs)
if ret[0].Interface() != false {
t.Fatalf("Got unexpected NeedsContainerd response")
}
case "GetLinuxContainerdURL":
rargs := make([]reflect.Value, 0)
ret := v.Call(rargs)
if ret[0].Interface() != "" {
t.Fatalf("Got unexpected NeedsContainerd response")
}
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions pkg/engine/templates_generated.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions test/e2e/cluster.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,8 @@ docker run --rm \
-e ARC_CLIENT_SECRET=${ARC_CLIENT_SECRET:-$AZURE_CLIENT_SECRET} \
-e ARC_SUBSCRIPTION_ID=${ARC_SUBSCRIPTION_ID:-$AZURE_SUBSCRIPTION_ID} \
-e ARC_LOCATION=${ARC_LOCATION:-$LOCATION} \
-e LINUX_CONTAINERD_URL=${LINUX_CONTAINERD_URL} \
-e WINDOWS_CONTAINERD_URL=${WINDOWS_CONTAINERD_URL} \
"${DEV_IMAGE}" make test-kubernetes || tryExit && renameResultsFile "deploy"

if [ "${UPGRADE_CLUSTER}" = "true" ] || [ "${SCALE_CLUSTER}" = "true" ] || [ -n "$ADD_NODE_POOL_INPUT" ] || [ "${GET_CLUSTER_LOGS}" = "true" ] || [ "${ROTATE_CERTS}" = "true" ]; then
Expand Down
10 changes: 10 additions & 0 deletions test/e2e/engine/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ type Config struct {
EnableTelemetry bool `envconfig:"ENABLE_TELEMETRY" default:"true"`
KubernetesImageBase string `envconfig:"KUBERNETES_IMAGE_BASE" default:""`
KubernetesImageBaseType string `envconfig:"KUBERNETES_IMAGE_BASE_TYPE" default:""`
LinuxContainerdURL string `envconfig:"LINUX_CONTAINERD_URL"`
WindowsContainerdURL string `envconfig:"WINDOWS_CONTAINERD_URL"`
*ArcOnboardingConfig

ClusterDefinitionPath string // The original template we want to use to build the cluster from.
Expand Down Expand Up @@ -366,6 +368,14 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetIDs []string, i
prop.OrchestratorProfile.KubernetesConfig.UserAssignedID = config.MSIUserAssignedID
}

if config.LinuxContainerdURL != "" {
prop.OrchestratorProfile.KubernetesConfig.LinuxContainerdURL = config.LinuxContainerdURL
}

if config.WindowsContainerdURL != "" {
prop.OrchestratorProfile.KubernetesConfig.WindowsContainerdURL = config.WindowsContainerdURL
}

return &Engine{
Config: config,
ClusterDefinition: cs,
Expand Down
1 change: 1 addition & 0 deletions test/e2e/test_cluster_configs/containerd.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
"kubernetesConfig": {
"networkPlugin": "azure",
"containerRuntime": "containerd",
"linuxContainerdURL": "https://packages.microsoft.com/repos/microsoft-ubuntu-bionic-prod/pool/main/m/moby-containerd/moby-containerd_1.3.7+azure-2_amd64.deb",
"windowsContainerdURL": "https://github.com/containerd/containerd/releases/download/v1.4.0/containerd-1.4.0-windows-amd64.tar.gz"
}
},
Expand Down

0 comments on commit 7dc01ab

Please sign in to comment.