From 97286c520b611e045e5129a0d4ffb2cdaffb4351 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Tue, 29 Nov 2022 01:32:26 +0200 Subject: [PATCH 01/10] add support for AliCloud RRSA auth Signed-off-by: Maxim Rubchinsky --- .../sdk/auth/credentials/oidc_credential.go | 37 +++ .../alibaba-cloud-sdk-go/sdk/auth/signer.go | 4 + .../sdk/auth/signers/signer_oidc.go | 247 ++++++++++++++++++ .../alibaba-cloud-sdk-go/sdk/client.go | 19 ++ .../services/ecs/client.go | 7 + .../services/ess/client.go | 7 + .../alicloud/alicloud_auto_scaling.go | 9 +- .../alicloud/alicloud_cloud_config.go | 53 +++- .../alicloud/alicloud_instance_types.go | 7 +- 9 files changed, 376 insertions(+), 14 deletions(-) create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials/oidc_credential.go create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials/oidc_credential.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials/oidc_credential.go new file mode 100644 index 000000000000..2d23de3cd98b --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials/oidc_credential.go @@ -0,0 +1,37 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credentials + +// OIDCCredential is a kind of credentials +type OIDCCredential struct { + RoleArn string + OIDCProviderArn string + OIDCTokenFilePath string + RoleSessionName string + RoleSessionExpiration int +} + +// NewOIDCRoleArnCredential returns OIDCCredential +func NewOIDCRoleArnCredential(roleArn, OIDCProviderArn, OIDCTokenFilePath, RoleSessionName string, RoleSessionExpiration int) *OIDCCredential { + return &OIDCCredential{ + RoleArn: roleArn, + OIDCProviderArn: OIDCProviderArn, + OIDCTokenFilePath: OIDCTokenFilePath, + RoleSessionName: RoleSessionName, + RoleSessionExpiration: RoleSessionExpiration, + } +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signer.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signer.go index 424db3b3cd3d..3f732a36f7f5 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signer.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signer.go @@ -61,6 +61,10 @@ func NewSignerWithCredential(credential Credential, commonApi func(request *requ { signer, err = signers.NewEcsRamRoleSigner(instance, commonApi) } + case *credentials.OIDCCredential: + { + signer, err = signers.NewOIDCSigner(instance) + } case *credentials.BaseCredential: // deprecated user interface { signer, err = signers.NewAccessKeySigner(instance.ToAccessKeyCredential()) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go new file mode 100644 index 000000000000..ccb1a2b41eb2 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -0,0 +1,247 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signers + +import ( + "encoding/json" + "fmt" + "github.com/google/uuid" + "github.com/jmespath/go-jmespath" + "io/ioutil" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/errors" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/requests" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/responses" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/utils" + "net/http" + "os" + "strconv" + "strings" + "time" +) + +const ( + defaultOIDCDurationSeconds = 3600 +) + +// OIDCSigner is kind of signer +type OIDCSigner struct { + *credentialUpdater + roleSessionName string + sessionCredential *SessionCredential + credential *credentials.OIDCCredential + commonApi func(request *requests.CommonRequest, signer interface{}) (response *responses.CommonResponse, err error) +} + +// NewOIDCSigner returns OIDCSigner +func NewOIDCSigner(credential *credentials.OIDCCredential) (signer *OIDCSigner, err error) { + signer = &OIDCSigner{ + credential: credential, + } + + signer.credentialUpdater = &credentialUpdater{ + credentialExpiration: credential.RoleSessionExpiration, + buildRequestMethod: signer.buildCommonRequest, + responseCallBack: signer.refreshCredential, + refreshApi: signer.refreshApi, + } + + if len(credential.RoleSessionName) > 0 { + signer.roleSessionName = credential.RoleSessionName + } else { + signer.roleSessionName = "aliyun-go-sdk-" + strconv.FormatInt(time.Now().UnixNano()/1000, 10) + } + if credential.RoleSessionExpiration > 0 { + if credential.RoleSessionExpiration >= 900 && credential.RoleSessionExpiration <= 3600 { + signer.credentialExpiration = credential.RoleSessionExpiration + } else { + err = errors.NewClientError(errors.InvalidParamErrorCode, "Assume Role session duration should be in the range of 15min - 1Hr", nil) + } + } else { + signer.credentialExpiration = defaultOIDCDurationSeconds + } + return +} + +// GetName returns "HMAC-SHA1" +func (*OIDCSigner) GetName() string { + return "HMAC-SHA1" +} + +// GetType returns "" +func (*OIDCSigner) GetType() string { + return "" +} + +// GetVersion returns "1.0" +func (*OIDCSigner) GetVersion() string { + return "1.0" +} + +// GetAccessKeyId returns accessKeyId +func (signer *OIDCSigner) GetAccessKeyId() (accessKeyId string, err error) { + if signer.sessionCredential == nil || signer.needUpdateCredential() { + err = signer.updateCredential() + } + if err != nil && (signer.sessionCredential == nil || len(signer.sessionCredential.AccessKeyId) <= 0) { + return "", err + } + return signer.sessionCredential.AccessKeyId, nil +} + +// GetExtraParam returns params +func (signer *OIDCSigner) GetExtraParam() map[string]string { + if signer.sessionCredential == nil || signer.needUpdateCredential() { + signer.updateCredential() + } + if signer.sessionCredential == nil || len(signer.sessionCredential.StsToken) <= 0 { + return make(map[string]string) + } + return map[string]string{"SecurityToken": signer.sessionCredential.StsToken} +} + +// Sign create signer +func (signer *OIDCSigner) Sign(stringToSign, secretSuffix string) string { + secret := signer.sessionCredential.AccessKeySecret + secretSuffix + return ShaHmac1(stringToSign, secret) +} + +func (signer *OIDCSigner) buildCommonRequest() (request *requests.CommonRequest, err error) { + request = requests.NewCommonRequest() + request.Domain = "sts.aliyuncs.com" + request.Scheme = requests.HTTPS + request.Method = "POST" + request.QueryParams["Timestamp"] = utils.GetTimeInFormatISO8601() + request.QueryParams["Action"] = "AssumeRoleWithOIDC" + request.QueryParams["Format"] = "JSON" + request.FormParams["RoleArn"] = signer.credential.RoleArn + request.FormParams["OIDCProviderArn"] = signer.credential.OIDCProviderArn + request.FormParams["OIDCToken"] = signer.getOIDCToken(signer.credential.OIDCTokenFilePath) + request.QueryParams["RoleSessionName"] = signer.credential.RoleSessionName + request.QueryParams["Version"] = "2015-04-01" + request.QueryParams["SignatureNonce"] = uuid.New().String() + request.Headers["Host"] = request.Domain + request.Headers["Accept-Encoding"] = "identity" + request.Headers["content-type"] = "application/x-www-form-urlencoded" + return +} + +func (signer *OIDCSigner) getOIDCToken(OIDCTokenFilePath string) string { + tokenPath := OIDCTokenFilePath + _, err := os.Stat(tokenPath) + if os.IsNotExist(err) { + tokenPath = os.Getenv("ALIBABA_CLOUD_OIDC_TOKEN_FILE") + if tokenPath == "" { + return "" + } + } + + token, err := ioutil.ReadFile(tokenPath) + if err != nil { + return "" + } + return string(token) +} + +func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response *responses.CommonResponse, err error) { + requestUrl := request.BuildUrl() + var urlEncoded string + if request.FormParams != nil { + urlEncoded = utils.GetUrlFormedMap(request.FormParams) + } + + httpRequest, err := http.NewRequest(request.Method, requestUrl, strings.NewReader(urlEncoded)) + if err != nil { + fmt.Println("refresh RRSA token err", err) + return + } + + httpRequest.Proto = "HTTP/1.1" + httpRequest.Host = request.Domain + for key, value := range request.Headers { + if value != "" { + httpRequest.Header[key] = []string{value} + } + } + + httpClient := &http.Client{} + httpResponse, err := httpClient.Do(httpRequest) + if err != nil { + fmt.Println("refresh RRSA token err", err) + return + } + + response = responses.NewCommonResponse() + err = responses.Unmarshal(response, httpResponse, "") + return +} + +func (signer *OIDCSigner) refreshCredential(response *responses.CommonResponse) (err error) { + if response.GetHttpStatus() != http.StatusOK { + message := "refresh session token failed" + err = errors.NewServerError(response.GetHttpStatus(), response.GetHttpContentString(), message) + return + } + var data interface{} + err = json.Unmarshal(response.GetHttpContentBytes(), &data) + if err != nil { + fmt.Println("refresh RRSA token err, json.Unmarshal fail", err) + return + } + accessKeyId, err := jmespath.Search("Credentials.AccessKeyId", data) + if err != nil { + fmt.Println("refresh RRSA token err, fail to get AccessKeyId", err) + return + } + accessKeySecret, err := jmespath.Search("Credentials.AccessKeySecret", data) + if err != nil { + fmt.Println("refresh RRSA token err, fail to get AccessKeySecret", err) + return + } + securityToken, err := jmespath.Search("Credentials.SecurityToken", data) + if err != nil { + fmt.Println("refresh RRSA token err, fail to get SecurityToken", err) + return + } + expiration, err := jmespath.Search("Credentials.Expiration", data) + if err != nil { + fmt.Println("refresh RRSA token err, fail to get Expiration", err) + return + } + + if accessKeyId == nil || accessKeySecret == nil || securityToken == nil { + return + } + + expirationTime, err := time.Parse("2006-01-02T15:04:05Z", expiration.(string)) + signer.credentialExpiration = int(expirationTime.Unix() - time.Now().Unix()) + signer.sessionCredential = &SessionCredential{ + AccessKeyId: accessKeyId.(string), + AccessKeySecret: accessKeySecret.(string), + StsToken: securityToken.(string), + } + + return +} + +// GetSessionCredential returns SessionCredential +func (signer *OIDCSigner) GetSessionCredential() *SessionCredential { + return signer.sessionCredential +} + +// Shutdown doesn't implement +func (signer *OIDCSigner) Shutdown() {} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client.go index 9ba0123d0475..9870f1210587 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client.go @@ -150,6 +150,18 @@ func (client *Client) InitWithEcsRamRole(regionId, roleName string) (err error) return client.InitWithOptions(regionId, config, credential) } +// InitWithRRSA need regionId,roleARN,oidcProviderARN,oidcTokenFilePath and roleSessionName +func (client *Client) InitWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName string) (err error) { + config := client.InitClientConfig() + credential := &credentials.OIDCCredential{ + RoleArn: roleARN, + OIDCProviderArn: oidcProviderARN, + OIDCTokenFilePath: oidcTokenFilePath, + RoleSessionName: roleSessionName, + } + return client.InitWithOptions(regionId, config, credential) +} + // InitClientConfig init client config func (client *Client) InitClientConfig() (config *Config) { if client.config != nil { @@ -395,6 +407,13 @@ func NewClientWithEcsRamRole(regionId string, roleName string) (client *Client, return } +// NewClientWithRRSA create client with RRSA on ECS +func NewClientWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName string) (client *Client, err error) { + client = &Client{} + err = client.InitWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName) + return +} + // NewClientWithRsaKeyPair create client with key-pair func NewClientWithRsaKeyPair(regionId string, publicKeyId, privateKey string, sessionExpiration int) (client *Client, err error) { client = &Client{} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ecs/client.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ecs/client.go index 3d43f1ff45df..170830bfe22f 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ecs/client.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ecs/client.go @@ -80,3 +80,10 @@ func NewClientWithRsaKeyPair(regionId string, publicKeyId, privateKey string, se err = client.InitWithRsaKeyPair(regionId, publicKeyId, privateKey, sessionExpiration) return } + +// NewClientWithRRSA is a shortcut to create sdk client with RRSA +func NewClientWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName string) (client *Client, err error) { + client = &Client{} + err = client.InitWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName) + return +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess/client.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess/client.go index cf19bb4bfdce..3be994bffe65 100755 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess/client.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess/client.go @@ -80,3 +80,10 @@ func NewClientWithRsaKeyPair(regionId string, publicKeyId, privateKey string, se err = client.InitWithRsaKeyPair(regionId, publicKeyId, privateKey, sessionExpiration) return } + +// NewClientWithRRSA is a shortcut to create sdk client with RRSA +func NewClientWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName string) (client *Client, err error) { + client = &Client{} + err = client.InitWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName) + return +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go index 8ccadf572478..3510ce3ab77b 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go @@ -52,7 +52,7 @@ func newAutoScalingWrapper(cfg *cloudConfig) (*autoScalingWrapper, error) { asw := &autoScalingWrapper{ cfg: cfg, } - if cfg.STSEnabled == true { + if cfg.STSEnabled || cfg.RRSAEnabled { go func(asw *autoScalingWrapper, cfg *cloudConfig) { timer := time.NewTicker(refreshClientInterval) defer timer.Stop() @@ -76,7 +76,7 @@ func newAutoScalingWrapper(cfg *cloudConfig) (*autoScalingWrapper, error) { func getEssClient(cfg *cloudConfig) (client *ess.Client, err error) { region := cfg.getRegion() - if cfg.STSEnabled == true { + if cfg.STSEnabled { auth, err := cfg.getSTSToken() if err != nil { klog.Errorf("Failed to get sts token from metadata,Because of %s", err.Error()) @@ -86,6 +86,11 @@ func getEssClient(cfg *cloudConfig) (client *ess.Client, err error) { if err != nil { klog.Errorf("Failed to create client with sts in metadata because of %s", err.Error()) } + } else if cfg.RRSAEnabled { + client, err = ess.NewClientWithRRSA(region, cfg.RoleARN, cfg.OIDCProviderARN, cfg.OIDCTokenFilePath, cfg.RoleSessionName) + if err != nil { + klog.Errorf("Failed to create ess client with RRSA, because of %s", err.Error()) + } } else { client, err = ess.NewClientWithAccessKey(region, cfg.AccessKeyID, cfg.AccessKeySecret) if err != nil { diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go index 36d518dac520..c547cc845482 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go @@ -23,16 +23,25 @@ import ( ) const ( - accessKeyId = "ACCESS_KEY_ID" - accessKeySecret = "ACCESS_KEY_SECRET" - regionId = "REGION_ID" + accessKeyId = "ACCESS_KEY_ID" + accessKeySecret = "ACCESS_KEY_SECRET" + oidcProviderARN = "ALICLOUD_OIDC_PROVIDER_ARN" + oidcTokenFilePath = "ALICLOUD_OIDC_TOKEN_FILE_PATH" + roleARN = "ALICLOUD_ROLE_ARN" + roleSessionName = "ALICLOUD_SESSION_NAME" + regionId = "REGION_ID" ) type cloudConfig struct { - RegionId string - AccessKeyID string - AccessKeySecret string - STSEnabled bool + RegionId string + AccessKeyID string + AccessKeySecret string + OIDCProviderARN string + OIDCTokenFilePath string + RoleARN string + RoleSessionName string + RRSAEnabled bool + STSEnabled bool } func (cc *cloudConfig) isValid() bool { @@ -48,18 +57,40 @@ func (cc *cloudConfig) isValid() bool { cc.RegionId = os.Getenv(regionId) } - if cc.RegionId == "" || cc.AccessKeyID == "" || cc.AccessKeySecret == "" { + if cc.OIDCProviderARN == "" { + cc.OIDCProviderARN = os.Getenv(oidcProviderARN) + } + + if cc.OIDCTokenFilePath == "" { + cc.OIDCTokenFilePath = os.Getenv(oidcTokenFilePath) + } + + if cc.RoleARN == "" { + cc.RoleARN = os.Getenv(roleARN) + } + + if cc.RoleSessionName == "" { + cc.RoleSessionName = os.Getenv(roleSessionName) + } + + if cc.RegionId != "" && cc.AccessKeyID != "" && cc.AccessKeySecret != "" { + klog.V(2).Info("Using AccessKey authentication") + return true + } else if cc.RegionId != "" && cc.OIDCProviderARN != "" && cc.OIDCTokenFilePath != "" && cc.RoleARN != "" && cc.RoleSessionName != "" { + klog.V(2).Info("Using RRSA authentication") + cc.RRSAEnabled = true + return true + } else { klog.V(5).Infof("Failed to get AccessKeyId:%s,AccessKeySecret:%s,RegionId:%s from cloudConfig and Env\n", cc.AccessKeyID, cc.AccessKeySecret, cc.RegionId) + klog.V(5).Infof("Failed to get OIDCProviderARN:%s,OIDCTokenFilePath:%s,RoleARN:%s,RoleSessionName:%s,RegionId:%s from cloudConfig and Env\n", cc.OIDCProviderARN, cc.OIDCTokenFilePath, cc.RoleARN, cc.RoleSessionName, cc.RegionId) klog.V(5).Infof("Try to use sts token in metadata instead.\n") if cc.validateSTSToken() == true && cc.getRegion() != "" { //if CA is working on ECS with valid role name, use sts token instead. cc.STSEnabled = true return true } - } else { - cc.STSEnabled = false - return true } + return false } diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go index 3f09394d78b9..e3c0e7afc3b5 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go @@ -107,7 +107,7 @@ func newInstanceWrapper(cfg *cloudConfig) (*instanceWrapper, error) { return nil, fmt.Errorf("your cloud config is not valid") } iw := &instanceWrapper{} - if cfg.STSEnabled == true { + if cfg.STSEnabled || cfg.RRSAEnabled { go func(iw *instanceWrapper, cfg *cloudConfig) { timer := time.NewTicker(refreshClientInterval) defer timer.Stop() @@ -141,6 +141,11 @@ func getEcsClient(cfg *cloudConfig) (client *ecs.Client, err error) { if err != nil { klog.Errorf("failed to create client with sts in metadata,because of %s", err.Error()) } + } else if cfg.RRSAEnabled { + client, err = ecs.NewClientWithRRSA(region, cfg.RoleARN, cfg.OIDCProviderARN, cfg.OIDCTokenFilePath, cfg.RoleSessionName) + if err != nil { + klog.Errorf("Failed to create ess client with RRSA, because of %s", err.Error()) + } } else { client, err = ecs.NewClientWithAccessKey(region, cfg.AccessKeyID, cfg.AccessKeySecret) if err != nil { From 49a9fe42a44539dc4e7b3d5c48d5873cdc18e878 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Tue, 29 Nov 2022 14:16:41 +0200 Subject: [PATCH 02/10] add support for AliCloud RRSA auth Signed-off-by: Maxim Rubchinsky --- .../cluster-autoscaler-rrsa-standard.yaml | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml diff --git a/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml new file mode 100644 index 000000000000..ab07b31e4ad8 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml @@ -0,0 +1,196 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["events","endpoints"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get","update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch","list","get","update"] +- apiGroups: [""] + resources: ["namespaces","pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] + verbs: ["watch","list","get"] +- apiGroups: ["extensions"] + resources: ["replicasets","daemonsets"] + verbs: ["watch","list","get"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch","list"] +- apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch","list","get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["watch","list","get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete","get","update","watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-config +type: Opaque +data: + oidc-provider-arn: [YOUR_BASE64_OIDC_PROVIDER_ARN] + oidc-token-file-path: [YOUR_BASE64_OIDC_TOKEN_FILE_PATH] + role-arn: [YOUR_BASE64_ROLE_ARN] + session-name: [YOUR_BASE64_SESSION_NAME] + region-id: [YOUR_BASE64_REGION_ID] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + priorityClassName: system-cluster-critical + serviceAccountName: cluster-autoscaler + containers: + - image: registry.cn-hangzhou.aliyuncs.com/acs/autoscaler:v1.3.1 + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=alicloud + - --nodes=[min]:[max]:[ASG_ID] + imagePullPolicy: "Always" + env: + - name: ALICLOUD_OIDC_PROVIDER_ARN + valueFrom: + secretKeyRef: + name: cloud-config + key: oidc-provider-arn + - name: ALICLOUD_OIDC_TOKEN_FILE_PATH + valueFrom: + secretKeyRef: + name: cloud-config + key: oidc-token-file-path + - name: ALICLOUD_ROLE_ARN + valueFrom: + secretKeyRef: + name: cloud-config + key: role-arn + - name: ALICLOUD_SESSION_NAME + valueFrom: + secretKeyRef: + name: cloud-config + key: session-name + - name: REGION_ID + valueFrom: + secretKeyRef: + name: cloud-config + key: region-id + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + - name: oidc-token + mountPath: /var/run/secrets/tokens + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + - name: oidc-token + projected: + sources: + - serviceAccountToken: + path: oidc-token + expirationSeconds: 7200 # The validity period of the OIDC token in seconds. + audience: "sts.aliyuncs.com" From 51ae336fb16665094b33b326ca91707e276644cf Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Tue, 29 Nov 2022 23:04:52 +0200 Subject: [PATCH 03/10] add support for AliCloud RRSA auth Signed-off-by: Maxim Rubchinsky --- .../sdk/auth/signers/signer_oidc.go | 42 +++++----- .../alicloud/alicloud_auto_scaling.go | 2 +- .../alicloud/alicloud_instance_types.go | 2 +- .../cluster-autoscaler-rrsa-standard.yaml | 81 ++++++++++--------- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index ccb1a2b41eb2..7b4c90f64b86 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -19,7 +19,6 @@ package signers import ( "encoding/json" "fmt" - "github.com/google/uuid" "github.com/jmespath/go-jmespath" "io/ioutil" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials" @@ -29,6 +28,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/utils" "net/http" "os" + "runtime" "strconv" "strings" "time" @@ -44,7 +44,6 @@ type OIDCSigner struct { roleSessionName string sessionCredential *SessionCredential credential *credentials.OIDCCredential - commonApi func(request *requests.CommonRequest, signer interface{}) (response *responses.CommonResponse, err error) } // NewOIDCSigner returns OIDCSigner @@ -100,6 +99,7 @@ func (signer *OIDCSigner) GetAccessKeyId() (accessKeyId string, err error) { if err != nil && (signer.sessionCredential == nil || len(signer.sessionCredential.AccessKeyId) <= 0) { return "", err } + return signer.sessionCredential.AccessKeyId, nil } @@ -111,6 +111,7 @@ func (signer *OIDCSigner) GetExtraParam() map[string]string { if signer.sessionCredential == nil || len(signer.sessionCredential.StsToken) <= 0 { return make(map[string]string) } + return map[string]string{"SecurityToken": signer.sessionCredential.StsToken} } @@ -121,22 +122,26 @@ func (signer *OIDCSigner) Sign(stringToSign, secretSuffix string) string { } func (signer *OIDCSigner) buildCommonRequest() (request *requests.CommonRequest, err error) { + const endpoint = "sts.aliyuncs.com" + const stsApiVersion = "2015-04-01" + const action = "AssumeRoleWithOIDC" request = requests.NewCommonRequest() - request.Domain = "sts.aliyuncs.com" request.Scheme = requests.HTTPS - request.Method = "POST" - request.QueryParams["Timestamp"] = utils.GetTimeInFormatISO8601() - request.QueryParams["Action"] = "AssumeRoleWithOIDC" + request.Domain = endpoint + request.Method = requests.POST + request.QueryParams["Action"] = action + request.QueryParams["Version"] = stsApiVersion request.QueryParams["Format"] = "JSON" + request.QueryParams["Timestamp"] = utils.GetTimeInFormatISO8601() + request.QueryParams["SignatureNonce"] = utils.GetUUIDV4() request.FormParams["RoleArn"] = signer.credential.RoleArn request.FormParams["OIDCProviderArn"] = signer.credential.OIDCProviderArn request.FormParams["OIDCToken"] = signer.getOIDCToken(signer.credential.OIDCTokenFilePath) request.QueryParams["RoleSessionName"] = signer.credential.RoleSessionName - request.QueryParams["Version"] = "2015-04-01" - request.QueryParams["SignatureNonce"] = uuid.New().String() - request.Headers["Host"] = request.Domain + request.Headers["host"] = endpoint request.Headers["Accept-Encoding"] = "identity" request.Headers["content-type"] = "application/x-www-form-urlencoded" + request.Headers["user-agent"] = fmt.Sprintf("AlibabaCloud (%s; %s) Golang/%s Core/%s TeaDSL/1", runtime.GOOS, runtime.GOARCH, strings.Trim(runtime.Version(), "go"), "0.01") return } @@ -158,13 +163,8 @@ func (signer *OIDCSigner) getOIDCToken(OIDCTokenFilePath string) string { } func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response *responses.CommonResponse, err error) { - requestUrl := request.BuildUrl() - var urlEncoded string - if request.FormParams != nil { - urlEncoded = utils.GetUrlFormedMap(request.FormParams) - } - - httpRequest, err := http.NewRequest(request.Method, requestUrl, strings.NewReader(urlEncoded)) + body := utils.GetUrlFormedMap(request.FormParams) + httpRequest, err := http.NewRequest(request.Method, fmt.Sprintf("%s://%s/?%s", strings.ToLower(request.Scheme), request.Domain, utils.GetUrlFormedMap(request.QueryParams)), strings.NewReader(body)) if err != nil { fmt.Println("refresh RRSA token err", err) return @@ -172,10 +172,8 @@ func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response httpRequest.Proto = "HTTP/1.1" httpRequest.Host = request.Domain - for key, value := range request.Headers { - if value != "" { - httpRequest.Header[key] = []string{value} - } + for k, v := range request.Headers { + httpRequest.Header.Add(k, v) } httpClient := &http.Client{} @@ -187,15 +185,17 @@ func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response response = responses.NewCommonResponse() err = responses.Unmarshal(response, httpResponse, "") + return } func (signer *OIDCSigner) refreshCredential(response *responses.CommonResponse) (err error) { if response.GetHttpStatus() != http.StatusOK { - message := "refresh session token failed" + message := "refresh RRSA failed" err = errors.NewServerError(response.GetHttpStatus(), response.GetHttpContentString(), message) return } + var data interface{} err = json.Unmarshal(response.GetHttpContentBytes(), &data) if err != nil { diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go index 3510ce3ab77b..4f974be0082a 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling.go @@ -52,7 +52,7 @@ func newAutoScalingWrapper(cfg *cloudConfig) (*autoScalingWrapper, error) { asw := &autoScalingWrapper{ cfg: cfg, } - if cfg.STSEnabled || cfg.RRSAEnabled { + if cfg.STSEnabled { go func(asw *autoScalingWrapper, cfg *cloudConfig) { timer := time.NewTicker(refreshClientInterval) defer timer.Stop() diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go index e3c0e7afc3b5..966d4b6ed00a 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types.go @@ -107,7 +107,7 @@ func newInstanceWrapper(cfg *cloudConfig) (*instanceWrapper, error) { return nil, fmt.Errorf("your cloud config is not valid") } iw := &instanceWrapper{} - if cfg.STSEnabled || cfg.RRSAEnabled { + if cfg.STSEnabled { go func(iw *instanceWrapper, cfg *cloudConfig) { timer := time.NewTicker(refreshClientInterval) defer timer.Stop() diff --git a/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml index ab07b31e4ad8..bd7d3220ad4c 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml +++ b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-rrsa-standard.yaml @@ -16,37 +16,49 @@ metadata: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler rules: -- apiGroups: [""] - resources: ["events","endpoints"] - verbs: ["create", "patch"] -- apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] -- apiGroups: [""] - resources: ["pods/status"] - verbs: ["update"] -- apiGroups: [""] - resources: ["endpoints"] - resourceNames: ["cluster-autoscaler"] - verbs: ["get","update"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["watch","list","get","update"] -- apiGroups: [""] - resources: ["namespaces","pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] - verbs: ["watch","list","get"] -- apiGroups: ["extensions"] - resources: ["replicasets","daemonsets"] - verbs: ["watch","list","get"] -- apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["watch","list"] -- apiGroups: ["apps"] - resources: ["statefulsets", "replicasets", "daemonsets"] - verbs: ["watch","list","get"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["watch","list","get"] + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "namespaces" + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "csistoragecapacities", "csidrivers"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -106,6 +118,7 @@ apiVersion: v1 kind: Secret metadata: name: cloud-config + namespace: kube-system type: Opaque data: oidc-provider-arn: [YOUR_BASE64_OIDC_PROVIDER_ARN] @@ -178,15 +191,9 @@ spec: name: cloud-config key: region-id volumeMounts: - - name: ssl-certs - mountPath: /etc/ssl/certs/ca-certificates.crt - readOnly: true - name: oidc-token mountPath: /var/run/secrets/tokens volumes: - - name: ssl-certs - hostPath: - path: "/etc/ssl/certs/ca-certificates.crt" - name: oidc-token projected: sources: From 3647ef66051bb61e673e8e782846e7c4e16046c4 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Sat, 15 Apr 2023 23:36:58 +0300 Subject: [PATCH 04/10] oidc clients tests Signed-off-by: Maxim Rubchinsky --- .../sdk/auth/signers/signer_oidc.go | 3 +- .../alibaba-cloud-sdk-go/sdk/client_test.go | 19 +++++++++++++ .../alicloud/alicloud_auto_scaling_test.go | 22 +++++++++++++++ .../alicloud/alicloud_cloud_config.go | 2 +- .../alicloud/alicloud_cloud_config_test.go | 28 +++++++++++++++++++ .../alicloud/alicloud_instance_types_test.go | 22 +++++++++++++++ 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go create mode 100644 cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index 7b4c90f64b86..6b9b1d590afc 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "github.com/jmespath/go-jmespath" - "io/ioutil" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/credentials" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/errors" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/requests" @@ -155,7 +154,7 @@ func (signer *OIDCSigner) getOIDCToken(OIDCTokenFilePath string) string { } } - token, err := ioutil.ReadFile(tokenPath) + token, err := os.ReadFile(tokenPath) if err != nil { return "" } diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go new file mode 100644 index 000000000000..d4ad1107ecde --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go @@ -0,0 +1,19 @@ +package sdk + +import ( + "github.com/stretchr/testify/assert" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers" + "testing" +) + +func TestRRSAClientInit(t *testing.T) { + oidcProviderARN := "acs:ram::12345:oidc-provider/ack-rrsa-cb123" + oidcTokenFilePath := "/var/run/secrets/tokens/oidc-token" + roleARN := "acs:ram::12345:role/autoscaler-role" + roleSessionName := "session" + regionId := "cn-hangzhou" + + client, err := NewClientWithRRSA(regionId, roleARN, oidcProviderARN, oidcTokenFilePath, roleSessionName) + assert.NoError(t, err) + assert.IsType(t, &signers.OIDCSigner{}, client.signer) +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go new file mode 100644 index 000000000000..92105abb6705 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go @@ -0,0 +1,22 @@ +package alicloud + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRRSACloudConfigEssClientCreation(t *testing.T) { + t.Setenv(oidcProviderARN, "acs:ram::12345:oidc-provider/ack-rrsa-cb123") + t.Setenv(oidcTokenFilePath, "/var/run/secrets/tokens/oidc-token") + t.Setenv(roleARN, "acs:ram::12345:role/autoscaler-role") + t.Setenv(roleSessionName, "session") + t.Setenv(regionId, "cn-hangzhou") + + cfg := &cloudConfig{} + assert.True(t, cfg.isValid()) + assert.True(t, cfg.RRSAEnabled) + + client, err := getEssClient(cfg) + assert.NoError(t, err) + assert.NotNil(t, client) +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go index c547cc845482..c6c41ebcc7c2 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go @@ -84,7 +84,7 @@ func (cc *cloudConfig) isValid() bool { klog.V(5).Infof("Failed to get AccessKeyId:%s,AccessKeySecret:%s,RegionId:%s from cloudConfig and Env\n", cc.AccessKeyID, cc.AccessKeySecret, cc.RegionId) klog.V(5).Infof("Failed to get OIDCProviderARN:%s,OIDCTokenFilePath:%s,RoleARN:%s,RoleSessionName:%s,RegionId:%s from cloudConfig and Env\n", cc.OIDCProviderARN, cc.OIDCTokenFilePath, cc.RoleARN, cc.RoleSessionName, cc.RegionId) klog.V(5).Infof("Try to use sts token in metadata instead.\n") - if cc.validateSTSToken() == true && cc.getRegion() != "" { + if cc.validateSTSToken() && cc.getRegion() != "" { //if CA is working on ECS with valid role name, use sts token instead. cc.STSEnabled = true return true diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go new file mode 100644 index 000000000000..a646669ead2a --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go @@ -0,0 +1,28 @@ +package alicloud + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAccessKeyCloudConfigIsValid(t *testing.T) { + t.Setenv(accessKeyId, "id") + t.Setenv(accessKeySecret, "secret") + t.Setenv(regionId, "cn-hangzhou") + + cfg := &cloudConfig{} + assert.True(t, cfg.isValid()) + assert.False(t, cfg.RRSAEnabled) +} + +func TestRRSACloudConfigIsValid(t *testing.T) { + t.Setenv(oidcProviderARN, "acs:ram::12345:oidc-provider/ack-rrsa-cb123") + t.Setenv(oidcTokenFilePath, "/var/run/secrets/tokens/oidc-token") + t.Setenv(roleARN, "acs:ram::12345:role/autoscaler-role") + t.Setenv(roleSessionName, "session") + t.Setenv(regionId, "cn-hangzhou") + + cfg := &cloudConfig{} + assert.True(t, cfg.isValid()) + assert.True(t, cfg.RRSAEnabled) +} diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go new file mode 100644 index 000000000000..b057e54a52bf --- /dev/null +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go @@ -0,0 +1,22 @@ +package alicloud + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRRSACloudConfigEcsClientCreation(t *testing.T) { + t.Setenv(oidcProviderARN, "acs:ram::12345:oidc-provider/ack-rrsa-cb123") + t.Setenv(oidcTokenFilePath, "/var/run/secrets/tokens/oidc-token") + t.Setenv(roleARN, "acs:ram::12345:role/autoscaler-role") + t.Setenv(roleSessionName, "session") + t.Setenv(regionId, "cn-hangzhou") + + cfg := &cloudConfig{} + assert.True(t, cfg.isValid()) + assert.True(t, cfg.RRSAEnabled) + + client, err := getEcsClient(cfg) + assert.NoError(t, err) + assert.NotNil(t, client) +} From 45c1d01e01400d1816e94233cf7115a49c6308cd Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Sun, 16 Apr 2023 15:22:45 +0300 Subject: [PATCH 05/10] headers Signed-off-by: Maxim Rubchinsky --- .../alibaba-cloud-sdk-go/sdk/client_test.go | 16 ++++++++++++++++ .../alicloud/alicloud_auto_scaling_test.go | 16 ++++++++++++++++ .../alicloud/alicloud_cloud_config_test.go | 16 ++++++++++++++++ .../alicloud/alicloud_instance_types_test.go | 16 ++++++++++++++++ 4 files changed, 64 insertions(+) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go index d4ad1107ecde..5b1ca237b5c1 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/client_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sdk import ( diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go index 92105abb6705..409fc4871b8a 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package alicloud import ( diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go index a646669ead2a..4c828367c5f0 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package alicloud import ( diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go index b057e54a52bf..eaf5cdeec516 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_instance_types_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package alicloud import ( From 4b899db5075ed5aa326b95c35ba0a007cf3ab159 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Mon, 24 Apr 2023 12:36:48 +0300 Subject: [PATCH 06/10] Update cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go Co-authored-by: Huang Huang --- .../cloudprovider/alicloud/alicloud_cloud_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go index c6c41ebcc7c2..1b9eb8db22d3 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go @@ -81,7 +81,7 @@ func (cc *cloudConfig) isValid() bool { cc.RRSAEnabled = true return true } else { - klog.V(5).Infof("Failed to get AccessKeyId:%s,AccessKeySecret:%s,RegionId:%s from cloudConfig and Env\n", cc.AccessKeyID, cc.AccessKeySecret, cc.RegionId) + klog.V(5).Infof("Failed to get AccessKeyId:%s,RegionId:%s from cloudConfig and Env\n", cc.AccessKeyID, cc.RegionId) klog.V(5).Infof("Failed to get OIDCProviderARN:%s,OIDCTokenFilePath:%s,RoleARN:%s,RoleSessionName:%s,RegionId:%s from cloudConfig and Env\n", cc.OIDCProviderARN, cc.OIDCTokenFilePath, cc.RoleARN, cc.RoleSessionName, cc.RegionId) klog.V(5).Infof("Try to use sts token in metadata instead.\n") if cc.validateSTSToken() && cc.getRegion() != "" { From 9e679dea8e0f2e216dd238cde79ec987a079da66 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Mon, 24 Apr 2023 12:37:40 +0300 Subject: [PATCH 07/10] Update cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go Co-authored-by: Huang Huang --- .../alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index 6b9b1d590afc..c45929d6ba85 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -61,7 +61,7 @@ func NewOIDCSigner(credential *credentials.OIDCCredential) (signer *OIDCSigner, if len(credential.RoleSessionName) > 0 { signer.roleSessionName = credential.RoleSessionName } else { - signer.roleSessionName = "aliyun-go-sdk-" + strconv.FormatInt(time.Now().UnixNano()/1000, 10) + signer.roleSessionName = "kubernetes-cluster-autoscaler-" + strconv.FormatInt(time.Now().UnixNano()/1000, 10) } if credential.RoleSessionExpiration > 0 { if credential.RoleSessionExpiration >= 900 && credential.RoleSessionExpiration <= 3600 { From 1c8ed21b69fddb1c00ac75e38e41cd02f69bfb13 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Mon, 24 Apr 2023 12:38:10 +0300 Subject: [PATCH 08/10] Update cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go Co-authored-by: Huang Huang --- .../alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index c45929d6ba85..3bc461cd8ffb 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -140,7 +140,7 @@ func (signer *OIDCSigner) buildCommonRequest() (request *requests.CommonRequest, request.Headers["host"] = endpoint request.Headers["Accept-Encoding"] = "identity" request.Headers["content-type"] = "application/x-www-form-urlencoded" - request.Headers["user-agent"] = fmt.Sprintf("AlibabaCloud (%s; %s) Golang/%s Core/%s TeaDSL/1", runtime.GOOS, runtime.GOARCH, strings.Trim(runtime.Version(), "go"), "0.01") + request.Headers["user-agent"] = fmt.Sprintf("AlibabaCloud (%s; %s) Golang/%s Core/%s TeaDSL/1 kubernetes-cluster-autoscaler", runtime.GOOS, runtime.GOARCH, strings.Trim(runtime.Version(), "go"), "0.01") return } From 933958c0456a5f1653c44bdab3b69a70637ecd00 Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Mon, 24 Apr 2023 12:47:24 +0300 Subject: [PATCH 09/10] Update cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go Co-authored-by: Huang Huang --- .../alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index 3bc461cd8ffb..b2a3c7b4ed89 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -156,6 +156,7 @@ func (signer *OIDCSigner) getOIDCToken(OIDCTokenFilePath string) string { token, err := os.ReadFile(tokenPath) if err != nil { + klog.Errorf("get oidc token from file %s failed: %s", tokenPath, err) return "" } return string(token) From 41ef6b3d3788da8a06e5e66ff7db51af4b9e544d Mon Sep 17 00:00:00 2001 From: Maxim Rubchinsky Date: Mon, 24 Apr 2023 12:57:07 +0300 Subject: [PATCH 10/10] cr fixes Signed-off-by: Maxim Rubchinsky --- .../sdk/auth/signers/signer_oidc.go | 18 ++++++++++-------- .../alicloud/alicloud_cloud_config.go | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go index b2a3c7b4ed89..930e9612e4dc 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/auth/signers/signer_oidc.go @@ -25,6 +25,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/requests" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/responses" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/sdk/utils" + "k8s.io/klog/v2" "net/http" "os" "runtime" @@ -150,13 +151,14 @@ func (signer *OIDCSigner) getOIDCToken(OIDCTokenFilePath string) string { if os.IsNotExist(err) { tokenPath = os.Getenv("ALIBABA_CLOUD_OIDC_TOKEN_FILE") if tokenPath == "" { + klog.Error("oidc token file path is missing") return "" } } token, err := os.ReadFile(tokenPath) if err != nil { - klog.Errorf("get oidc token from file %s failed: %s", tokenPath, err) + klog.Errorf("get oidc token from file %s failed: %s", tokenPath, err) return "" } return string(token) @@ -166,7 +168,7 @@ func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response body := utils.GetUrlFormedMap(request.FormParams) httpRequest, err := http.NewRequest(request.Method, fmt.Sprintf("%s://%s/?%s", strings.ToLower(request.Scheme), request.Domain, utils.GetUrlFormedMap(request.QueryParams)), strings.NewReader(body)) if err != nil { - fmt.Println("refresh RRSA token err", err) + klog.Errorf("refresh RRSA token failed: %s", err) return } @@ -179,7 +181,7 @@ func (signer *OIDCSigner) refreshApi(request *requests.CommonRequest) (response httpClient := &http.Client{} httpResponse, err := httpClient.Do(httpRequest) if err != nil { - fmt.Println("refresh RRSA token err", err) + klog.Errorf("refresh RRSA token failed: %s", err) return } @@ -199,27 +201,27 @@ func (signer *OIDCSigner) refreshCredential(response *responses.CommonResponse) var data interface{} err = json.Unmarshal(response.GetHttpContentBytes(), &data) if err != nil { - fmt.Println("refresh RRSA token err, json.Unmarshal fail", err) + klog.Errorf("refresh RRSA token err, json.Unmarshal fail: %s", err) return } accessKeyId, err := jmespath.Search("Credentials.AccessKeyId", data) if err != nil { - fmt.Println("refresh RRSA token err, fail to get AccessKeyId", err) + klog.Errorf("refresh RRSA token err, fail to get AccessKeyId: %s", err) return } accessKeySecret, err := jmespath.Search("Credentials.AccessKeySecret", data) if err != nil { - fmt.Println("refresh RRSA token err, fail to get AccessKeySecret", err) + klog.Errorf("refresh RRSA token err, fail to get AccessKeySecret: %s", err) return } securityToken, err := jmespath.Search("Credentials.SecurityToken", data) if err != nil { - fmt.Println("refresh RRSA token err, fail to get SecurityToken", err) + klog.Errorf("refresh RRSA token err, fail to get SecurityToken: %s", err) return } expiration, err := jmespath.Search("Credentials.Expiration", data) if err != nil { - fmt.Println("refresh RRSA token err, fail to get Expiration", err) + klog.Errorf("refresh RRSA token err, fail to get Expiration: %s", err) return } diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go index 1b9eb8db22d3..9a624459163d 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_cloud_config.go @@ -18,7 +18,7 @@ package alicloud import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/metadata" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" "os" )