diff --git a/api/v1alpha2/azurecluster_conversion.go b/api/v1alpha2/azurecluster_conversion.go index 55ad1ef0eb6..b7cfb45147b 100644 --- a/api/v1alpha2/azurecluster_conversion.go +++ b/api/v1alpha2/azurecluster_conversion.go @@ -61,6 +61,7 @@ func (src *AzureCluster) ConvertTo(dstRaw conversion.Hub) error { // nolint dst.Status.FailureDomains = restored.Status.FailureDomains dst.Spec.NetworkSpec.Vnet.CIDRBlocks = restored.Spec.NetworkSpec.Vnet.CIDRBlocks + dst.Spec.IdentityRef = restored.Spec.IdentityRef for _, restoredSubnet := range restored.Spec.NetworkSpec.Subnets { if restoredSubnet != nil { diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 52e1f428441..e1be7a829ea 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -384,6 +384,7 @@ func autoConvert_v1alpha3_AzureClusterSpec_To_v1alpha2_AzureClusterSpec(in *v1al out.Location = in.Location // WARNING: in.ControlPlaneEndpoint requires manual conversion: does not exist in peer-type out.AdditionalTags = *(*Tags)(unsafe.Pointer(&in.AdditionalTags)) + // WARNING: in.IdentityRef requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1alpha3/azurecluster_types.go b/api/v1alpha3/azurecluster_types.go index fd05f6a00b7..88f4ff45b22 100644 --- a/api/v1alpha3/azurecluster_types.go +++ b/api/v1alpha3/azurecluster_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha3 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" ) @@ -25,6 +26,9 @@ const ( // ClusterFinalizer allows ReconcileAzureCluster to clean up Azure resources associated with AzureCluster before // removing it from the apiserver. ClusterFinalizer = "azurecluster.infrastructure.cluster.x-k8s.io" + + // ClusterLabelNamespace indicates the namespace of the cluster + ClusterLabelNamespace = "azurecluster.infrastructure.cluster.x-k8s.io/cluster-namespace" ) // AzureClusterSpec defines the desired state of AzureCluster @@ -48,6 +52,10 @@ type AzureClusterSpec struct { // ones added by default. // +optional AdditionalTags Tags `json:"additionalTags,omitempty"` + + // IdentityRef is a reference to a AzureIdentity to be used when reconciling this cluster + // +optional + IdentityRef *corev1.ObjectReference `json:"identityRef,omitempty"` } // AzureClusterStatus defines the observed state of AzureCluster diff --git a/api/v1alpha3/azureclusteridentity_types.go b/api/v1alpha3/azureclusteridentity_types.go new file mode 100644 index 00000000000..c8ea3a251dd --- /dev/null +++ b/api/v1alpha3/azureclusteridentity_types.go @@ -0,0 +1,105 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" +) + +// AzureClusterIdentitySpec defines the parameters that are used to create an AzureIdentity +type AzureClusterIdentitySpec struct { + // UserAssignedMSI or Service Principal + Type IdentityType `json:"type"` + // User assigned MSI resource id. + // +optional + ResourceID string `json:"resourceID,omitempty"` + // Both User Assigned MSI and SP can use this field. + ClientID string `json:"clientID"` + // ClientSecret is a secret reference which should contain either a Service Principal password or certificate secret. + // +optional + ClientSecret corev1.SecretReference `json:"clientSecret,omitempty"` + // Service principal primary tenant id. + TenantID string `json:"tenantID"` + // AllowedNamespaces is an array of namespaces that AzureClusters can + // use this Identity from. + // + // An empty list (default) indicates that AzureClusters can use this + // Identity from any namespace. This field is intentionally not a + // pointer because the nil behavior (no namespaces) is undesirable here. + // +optional + AllowedNamespaces []string `json:"allowedNamespaces"` +} + +// AzureClusterIdentityStatus defines the observed state of AzureClusterIdentity +type AzureClusterIdentityStatus struct { + // Conditions defines current service state of the AzureClusterIdentity. + // +optional + Conditions clusterv1.Conditions `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=azureclusteridentities,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:subresource:status + +// AzureClusterIdentity is the Schema for the azureclustersidentities API +type AzureClusterIdentity struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AzureClusterIdentitySpec `json:"spec,omitempty"` + Status AzureClusterIdentityStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// AzureClusterIdentityList contains a list of AzureClusterIdentity +type AzureClusterIdentityList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AzureClusterIdentity `json:"items"` +} + +// GetConditions returns the list of conditions for an AzureClusterIdentity API object. +func (c *AzureClusterIdentity) GetConditions() clusterv1.Conditions { + return c.Status.Conditions +} + +// SetConditions will set the given conditions on an AzureClusterIdentity object +func (c *AzureClusterIdentity) SetConditions(conditions clusterv1.Conditions) { + c.Status.Conditions = conditions +} + +// ClusterNamespaceAllowed indicates if the cluster namespace is allowed +func (c *AzureClusterIdentity) ClusterNamespaceAllowed(namespace string) bool { + if len(c.Spec.AllowedNamespaces) == 0 { + return true + } + for _, v := range c.Spec.AllowedNamespaces { + if v == namespace { + return true + } + } + + return false +} + +func init() { + SchemeBuilder.Register(&AzureClusterIdentity{}, &AzureClusterIdentityList{}) +} diff --git a/api/v1alpha3/azureclusteridentity_types_test.go b/api/v1alpha3/azureclusteridentity_types_test.go new file mode 100644 index 00000000000..0617504186c --- /dev/null +++ b/api/v1alpha3/azureclusteridentity_types_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestAllowedNamespaces(t *testing.T) { + g := NewWithT(t) + + tests := []struct { + name string + identity *AzureClusterIdentity + clusterNamespace string + expected bool + }{ + { + name: "allow any cluster namespace when empty", + identity: &AzureClusterIdentity{ + Spec: AzureClusterIdentitySpec{ + AllowedNamespaces: []string{}, + }, + }, + clusterNamespace: "default", + expected: true, + }, + { + name: "allow cluster with namespace in list", + identity: &AzureClusterIdentity{ + Spec: AzureClusterIdentitySpec{ + AllowedNamespaces: []string{"namespace24", "namespace32"}, + }, + }, + clusterNamespace: "namespace24", + expected: true, + }, + { + name: "don't allow cluster with namespace not in list", + identity: &AzureClusterIdentity{ + Spec: AzureClusterIdentitySpec{ + AllowedNamespaces: []string{"namespace24", "namespace32"}, + }, + }, + clusterNamespace: "namespace8", + expected: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := tc.identity.ClusterNamespaceAllowed(tc.clusterNamespace) + g.Expect(actual).To(Equal(tc.expected)) + }) + } +} diff --git a/api/v1alpha3/conditions_consts.go b/api/v1alpha3/conditions_consts.go index 2cd86493b43..4328501d0e3 100644 --- a/api/v1alpha3/conditions_consts.go +++ b/api/v1alpha3/conditions_consts.go @@ -26,6 +26,8 @@ const ( LoadBalancerProvisioningReason = "LoadBalancerProvisioning" // LoadBalancerProvisioningFailedReason used for failure during provisioning of loadbalancer. LoadBalancerProvisioningFailedReason = "LoadBalancerProvisioningFailed" + // NamespaceNotAllowedByIdentity used to indicate cluster in a namespace not allowed by identity + NamespaceNotAllowedByIdentity = "NamespaceNotAllowedByIdentity" ) // AzureMachine Conditions and Reasons diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index 96242031e34..b03b844f42f 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -21,6 +21,8 @@ import ( ) const ( + // ControllerNamespace is the namespace where controller manager will run + ControllerNamespace = "capz-system" // ControlPlane machine label ControlPlane string = "control-plane" // Node machine label @@ -321,6 +323,25 @@ type UserAssignedIdentity struct { ProviderID string `json:"providerID"` } +const ( + // AzureIdentityBindingSelector is the label used to match with the AzureIdentityBinding + // For the controller to match an identity binding, it needs a [label] with the key `aadpodidbinding` + // whose value is that of the `selector:` field in the `AzureIdentityBinding`. + AzureIdentityBindingSelector = "capz-controller-aadpodidentity-selector" +) + +// IdentityType represents different types of identities. +// +kubebuilder:validation:Enum=ServicePrincipal;UserAssignedMSI +type IdentityType string + +const ( + // UserAssignedMSI represents a user-assigned identity. + UserAssignedMSI IdentityType = "UserAssignedMSI" + + // ServicePrincipal represents a service principal. + ServicePrincipal IdentityType = "ServicePrincipal" +) + // OSDisk defines the operating system disk for a VM. type OSDisk struct { OSType string `json:"osType"` diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index e8da52136d3..fd3d0111dc3 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -94,6 +94,108 @@ func (in *AzureCluster) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureClusterIdentity) DeepCopyInto(out *AzureClusterIdentity) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureClusterIdentity. +func (in *AzureClusterIdentity) DeepCopy() *AzureClusterIdentity { + if in == nil { + return nil + } + out := new(AzureClusterIdentity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureClusterIdentity) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureClusterIdentityList) DeepCopyInto(out *AzureClusterIdentityList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AzureClusterIdentity, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureClusterIdentityList. +func (in *AzureClusterIdentityList) DeepCopy() *AzureClusterIdentityList { + if in == nil { + return nil + } + out := new(AzureClusterIdentityList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AzureClusterIdentityList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureClusterIdentitySpec) DeepCopyInto(out *AzureClusterIdentitySpec) { + *out = *in + out.ClientSecret = in.ClientSecret + if in.AllowedNamespaces != nil { + in, out := &in.AllowedNamespaces, &out.AllowedNamespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureClusterIdentitySpec. +func (in *AzureClusterIdentitySpec) DeepCopy() *AzureClusterIdentitySpec { + if in == nil { + return nil + } + out := new(AzureClusterIdentitySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureClusterIdentityStatus) DeepCopyInto(out *AzureClusterIdentityStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(apiv1alpha3.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureClusterIdentityStatus. +func (in *AzureClusterIdentityStatus) DeepCopy() *AzureClusterIdentityStatus { + if in == nil { + return nil + } + out := new(AzureClusterIdentityStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureClusterList) DeepCopyInto(out *AzureClusterList) { *out = *in @@ -138,6 +240,11 @@ func (in *AzureClusterSpec) DeepCopyInto(out *AzureClusterSpec) { (*out)[key] = val } } + if in.IdentityRef != nil { + in, out := &in.IdentityRef, &out.IdentityRef + *out = new(v1.ObjectReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureClusterSpec. diff --git a/cloud/scope/clients.go b/cloud/scope/clients.go index 83a3f248577..84a814e5a48 100644 --- a/cloud/scope/clients.go +++ b/cloud/scope/clients.go @@ -17,6 +17,7 @@ limitations under the License. package scope import ( + "context" "fmt" "strings" @@ -83,3 +84,29 @@ func (c *AzureClients) setCredentials(subscriptionID string) error { c.Authorizer, err = c.GetAuthorizer() return err } + +func (c *AzureClients) setCredentialsWithProvider(ctx context.Context, subscriptionID string, credentialsProvider *AzureCredentialsProvider) error { + if credentialsProvider == nil { + return fmt.Errorf("Credentials provider cannot have an empty value") + } + + settings, err := auth.GetSettingsFromEnvironment() + if err != nil { + return err + } + + if subscriptionID == "" { + subscriptionID = settings.GetSubscriptionID() + if subscriptionID == "" { + return fmt.Errorf("error creating azure services. subscriptionID is not set in cluster or AZURE_SUBSCRIPTION_ID env var") + } + } + + c.EnvironmentSettings = settings + c.ResourceManagerEndpoint = settings.Environment.ResourceManagerEndpoint + c.ResourceManagerVMDNSSuffix = settings.Environment.ResourceManagerVMDNSSuffix + c.Values[auth.SubscriptionID] = strings.TrimSuffix(subscriptionID, "\n") + + c.Authorizer, err = credentialsProvider.GetAuthorizer(ctx, c.ResourceManagerEndpoint) + return err +} diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 8d07b31862a..9efc4833970 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -49,7 +49,7 @@ type ClusterScopeParams struct { // NewClusterScope creates a new Scope from the supplied parameters. // This is meant to be called for each reconcile iteration. -func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { +func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterScope, error) { if params.Cluster == nil { return nil, errors.New("failed to generate new scope from nil Cluster") } @@ -61,9 +61,20 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { params.Logger = klogr.New() } - err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID) - if err != nil { - return nil, err + if params.AzureCluster.Spec.IdentityRef == nil { + err := params.AzureClients.setCredentials(params.AzureCluster.Spec.SubscriptionID) + if err != nil { + return nil, errors.Wrap(err, "failed to configure azure settings and credentials from environment") + } + } else { + credentailsProvider, err := NewAzureCredentialsProvider(ctx, params.Client, params.AzureCluster) + if err != nil { + return nil, errors.Wrap(err, "failed to init credentials provider") + } + err = params.AzureClients.setCredentialsWithProvider(ctx, params.AzureCluster.Spec.SubscriptionID, credentailsProvider) + if err != nil { + return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity") + } } helper, err := patch.NewHelper(params.AzureCluster, params.Client) diff --git a/cloud/scope/identity.go b/cloud/scope/identity.go new file mode 100644 index 00000000000..71acdf08689 --- /dev/null +++ b/cloud/scope/identity.go @@ -0,0 +1,162 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + "context" + "fmt" + + "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/adal" + "github.com/pkg/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/identity" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + clusterctl "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" + "sigs.k8s.io/controller-runtime/pkg/client" + + aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// AzureCredentialsProvider provides +type AzureCredentialsProvider struct { + Client client.Client + AzureCluster *infrav1.AzureCluster + Identity *infrav1.AzureClusterIdentity +} + +// NewAzureCredentialsProvider creates a new AzureCredentialsProvider from the supplied inputs. +func NewAzureCredentialsProvider(ctx context.Context, kubeClient client.Client, azureCluster *infrav1.AzureCluster) (*AzureCredentialsProvider, error) { + if azureCluster.Spec.IdentityRef == nil { + return nil, errors.New("failed to generate new AzureCredentialsProvider from empty identityName") + } + + ref := azureCluster.Spec.IdentityRef + // if the namespace isn't specified then assume it's in the same namespace as the AzureCluster + namespace := ref.Namespace + if namespace == "" { + namespace = azureCluster.Namespace + } + identity := &infrav1.AzureClusterIdentity{} + key := client.ObjectKey{Name: ref.Name, Namespace: namespace} + if err := kubeClient.Get(ctx, key, identity); err != nil { + return nil, errors.Errorf("failed to retrieve AzureClusterIdentity external object %q/%q: %v", key.Namespace, key.Name, err) + } + + if identity.Spec.Type != infrav1.ServicePrincipal { + return nil, errors.New("AzureClusterIdentity is not of type Service Principal") + } + + return &AzureCredentialsProvider{ + Client: kubeClient, + AzureCluster: azureCluster, + Identity: identity, + }, nil +} + +// GetAuthorizer returns an Azure authorizer based on the provided azure identity +func (p *AzureCredentialsProvider) GetAuthorizer(ctx context.Context, resourceManagerEndpoint string) (autorest.Authorizer, error) { + azureIdentityType, err := getAzureIdentityType(p.Identity) + if err != nil { + return nil, err + } + copiedIdentity := &aadpodv1.AzureIdentity{ + TypeMeta: metav1.TypeMeta{ + Kind: "AzureIdentity", + APIVersion: "aadpodidentity.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: identity.GetAzureIdentityName(p.AzureCluster.Name, p.AzureCluster.Namespace, p.Identity.Name), + Namespace: infrav1.ControllerNamespace, + Annotations: map[string]string{ + aadpodv1.BehaviorKey: "namespaced", + }, + Labels: map[string]string{ + clusterv1.ClusterLabelName: p.AzureCluster.Name, + infrav1.ClusterLabelNamespace: p.AzureCluster.Namespace, + clusterctl.ClusterctlMoveLabelName: "true", + }, + OwnerReferences: p.AzureCluster.OwnerReferences, + }, + Spec: aadpodv1.AzureIdentitySpec{ + Type: azureIdentityType, + TenantID: p.Identity.Spec.TenantID, + ClientID: p.Identity.Spec.ClientID, + ClientPassword: p.Identity.Spec.ClientSecret, + ResourceID: p.Identity.Spec.ResourceID, + }, + } + err = p.Client.Create(ctx, copiedIdentity) + if err != nil && !apierrors.IsAlreadyExists(err) { + return nil, errors.Errorf("failed to create copied AzureIdentity %s in %s: %v", copiedIdentity.Name, infrav1.ControllerNamespace, err) + } + + azureIdentityBinding := &aadpodv1.AzureIdentityBinding{ + TypeMeta: metav1.TypeMeta{ + Kind: "AzureIdentityBinding", + APIVersion: "aadpodidentity.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-binding", copiedIdentity.Name), + Namespace: copiedIdentity.Namespace, + Labels: map[string]string{ + clusterv1.ClusterLabelName: p.AzureCluster.Name, + infrav1.ClusterLabelNamespace: p.AzureCluster.Namespace, + clusterctl.ClusterctlMoveLabelName: "true", + }, + OwnerReferences: p.AzureCluster.OwnerReferences, + }, + Spec: aadpodv1.AzureIdentityBindingSpec{ + AzureIdentity: copiedIdentity.Name, + Selector: infrav1.AzureIdentityBindingSelector, //should be same as selector added on controller + }, + } + err = p.Client.Create(ctx, azureIdentityBinding) + if err != nil && !apierrors.IsAlreadyExists(err) { + return nil, errors.Errorf("failed to create AzureIdentityBinding %s in %s: %v", copiedIdentity.Name, infrav1.ControllerNamespace, err) + } + + var spt *adal.ServicePrincipalToken + msiEndpoint, err := adal.GetMSIVMEndpoint() + if err != nil { + return nil, errors.Errorf("failed to get MSI endpoint: %v", err) + } + if p.Identity.Spec.Type == infrav1.ServicePrincipal { + spt, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, resourceManagerEndpoint, p.Identity.Spec.ClientID) + if err != nil { + return nil, errors.Errorf("failed to get token from service principal identity: %v", err) + } + } else if p.Identity.Spec.Type == infrav1.UserAssignedMSI { + return nil, errors.Errorf("UserAssignedMSI not supported: %v", err) + } + + return autorest.NewBearerAuthorizer(spt), nil +} + +func getAzureIdentityType(identity *infrav1.AzureClusterIdentity) (aadpodv1.IdentityType, error) { + switch identity.Spec.Type { + case infrav1.ServicePrincipal: + return aadpodv1.ServicePrincipal, nil + case infrav1.UserAssignedMSI: + return aadpodv1.UserAssignedMSI, nil + } + + return 0, errors.New("AzureIdentity does not have a vaild type") + +} diff --git a/cloud/services/disks/disks_test.go b/cloud/services/disks/disks_test.go index fb81c32e91c..b2fc3c7aa17 100644 --- a/cloud/services/disks/disks_test.go +++ b/cloud/services/disks/disks_test.go @@ -224,7 +224,7 @@ func TestDiskSpecs(t *testing.T) { azureMachine, } client := fake.NewFakeClientWithScheme(scheme, initObjects...) - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.Background(), scope.ClusterScopeParams{ AzureClients: scope.AzureClients{ Authorizer: autorest.NullAuthorizer{}, }, diff --git a/cloud/services/scalesets/scalesets_test.go b/cloud/services/scalesets/scalesets_test.go index 9b3d76fc6fb..8d60085de11 100644 --- a/cloud/services/scalesets/scalesets_test.go +++ b/cloud/services/scalesets/scalesets_test.go @@ -52,7 +52,7 @@ func TestNewService(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, } client := fake.NewFakeClientWithScheme(scheme.Scheme, cluster) - s, err := scope.NewClusterScope(scope.ClusterScopeParams{ + s, err := scope.NewClusterScope(context.Background(), scope.ClusterScopeParams{ AzureClients: scope.AzureClients{ Authorizer: autorest.NullAuthorizer{}, }, diff --git a/config/aadpodidentity/aad-pod-identity-deployment.yaml b/config/aadpodidentity/aad-pod-identity-deployment.yaml new file mode 100644 index 00000000000..dcdec1d12f9 --- /dev/null +++ b/config/aadpodidentity/aad-pod-identity-deployment.yaml @@ -0,0 +1,144 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azureidentitybindings.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + version: v1 + names: + kind: AzureIdentityBinding + plural: azureidentitybindings + scope: Namespaced +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azureidentities.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + version: v1 + names: + kind: AzureIdentity + singular: azureidentity + plural: azureidentities + scope: Namespaced +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azurepodidentityexceptions.aadpodidentity.k8s.io +spec: + group: aadpodidentity.k8s.io + version: v1 + names: + kind: AzurePodIdentityException + singular: azurepodidentityexception + plural: azurepodidentityexceptions + scope: Namespaced +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aad-pod-id-nmi-role +rules: +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +- apiGroups: ["aadpodidentity.k8s.io"] + resources: ["azureidentitybindings", "azureidentities", "azurepodidentityexceptions"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aad-pod-id-nmi-binding + labels: + k8s-app: capz-aad-pod-id-nmi-binding +subjects: +- kind: ServiceAccount + name: default + namespace: capz-system +roleRef: + kind: ClusterRole + name: aad-pod-id-nmi-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + component: nmi + tier: node + k8s-app: aad-pod-id + name: nmi + namespace: capz-system +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + component: nmi + tier: node + template: + metadata: + labels: + component: nmi + tier: node + spec: + serviceAccountName: default + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: iptableslock + containers: + - name: nmi + image: "mcr.microsoft.com/oss/azure/aad-pod-identity/nmi:v1.6.3" + imagePullPolicy: Always + args: + - "--node=$(NODE_NAME)" + - "--forceNamespaced" + - "--http-probe-port=8085" + - "--operation-mode=managed" + env: + - name: FORCENAMESPACED + value: "true" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + securityContext: + capabilities: + add: + - NET_ADMIN + volumeMounts: + - mountPath: /run/xtables.lock + name: iptableslock + livenessProbe: + httpGet: + path: /healthz + port: 8085 + initialDelaySeconds: 10 + periodSeconds: 5 + nodeSelector: + kubernetes.io/os: linux diff --git a/config/aadpodidentity/kustomization.yaml b/config/aadpodidentity/kustomization.yaml new file mode 100644 index 00000000000..1f7f3a7f049 --- /dev/null +++ b/config/aadpodidentity/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - aad-pod-identity-deployment.yaml diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml new file mode 100644 index 00000000000..333f602921b --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml @@ -0,0 +1,144 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: azureclusteridentities.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureClusterIdentity + listKind: AzureClusterIdentityList + plural: azureclusteridentities + singular: azureclusteridentity + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureClusterIdentity is the Schema for the azureclustersidentities + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AzureClusterIdentitySpec defines the parameters that are + used to create an AzureIdentity + properties: + allowedNamespaces: + description: "AllowedNamespaces is an array of namespaces that AzureClusters + can use this Identity from. \n An empty list (default) indicates + that AzureClusters can use this Identity from any namespace. This + field is intentionally not a pointer because the nil behavior (no + namespaces) is undesirable here." + items: + type: string + type: array + clientID: + description: Both User Assigned MSI and SP can use this field. + type: string + clientSecret: + description: ClientSecret is a secret reference which should contain + either a Service Principal password or certificate secret. + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + resourceID: + description: User assigned MSI resource id. + type: string + tenantID: + description: Service principal primary tenant id. + type: string + type: + description: UserAssignedMSI or Service Principal + enum: + - ServicePrincipal + - UserAssignedMSI + type: string + required: + - clientID + - tenantID + - type + type: object + status: + description: AzureClusterIdentityStatus defines the observed state of + AzureClusterIdentity + properties: + conditions: + description: Conditions defines current service state of the AzureClusterIdentity. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index 12c33351d81..aaf7082d5e1 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -459,6 +459,43 @@ spec: - host - port type: object + identityRef: + description: IdentityRef is a reference to a AzureIdentity to be used + when reconciling this cluster + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object location: type: string networkSpec: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureserviceprincipals.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureserviceprincipals.yaml new file mode 100644 index 00000000000..1774dc88b20 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureserviceprincipals.yaml @@ -0,0 +1,122 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: azureserviceprincipals.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureServicePrincipal + listKind: AzureServicePrincipalList + plural: azureserviceprincipals + singular: azureserviceprincipal + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureServicePrincipal represents a reference to an Azure access + key ID and secret access key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AzureServicePrincipalSpec. + properties: + allowedNamespaces: + description: "AllowedNamespaces is a selector of namespaces that AzureClusters + can use this ClusterPrincipal from. This is a standard Kubernetes + LabelSelector, a label query over a set of resources. The result + of matchLabels and matchExpressions are ANDed. Controllers must + not support AzureClusters in namespaces outside this selector. \n + An empty selector (default) indicates that AzureClusters can use + this AzureServicePrincipal from any namespace. This field is intentionally + not a pointer because the nil behavior (no namespaces) is undesirable + here." + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + name: + type: string + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: tenantID clientID + clientSecret' + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + required: + - name + - secretRef + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuresystemassignedidentites.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuresystemassignedidentites.yaml new file mode 100644 index 00000000000..ea2d1f99b5b --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuresystemassignedidentites.yaml @@ -0,0 +1,121 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: azuresystemassignedidentites.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureSystemAssignedIdentity + listKind: AzureSystemAssignedIdentityList + plural: azuresystemassignedidentites + singular: azuresystemassignedidentity + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureSystemAssignedIdentity represents a reference to an Azure + access key ID and secret access key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AzureSystemAssignedIdentitySpec. + properties: + allowedNamespaces: + description: "AllowedNamespaces is a selector of namespaces that AzureClusters + can use this ClusterPrincipal from. This is a standard Kubernetes + LabelSelector, a label query over a set of resources. The result + of matchLabels and matchExpressions are ANDed. Controllers must + not support AzureClusters in namespaces outside this selector. \n + An empty selector (default) indicates that AzureClusters can use + this AzureSystemAssignedIdentity from any namespace. This field + is intentionally not a pointer because the nil behavior (no namespaces) + is undesirable here." + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + name: + type: string + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: tenantID' + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + required: + - name + - secretRef + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureuserassignedidentites.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureuserassignedidentites.yaml new file mode 100644 index 00000000000..d7dec8ed8fd --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureuserassignedidentites.yaml @@ -0,0 +1,121 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + creationTimestamp: null + name: azureuserassignedidentites.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AzureUserAssignedIdentity + listKind: AzureUserAssignedIdentityList + plural: azureuserassignedidentites + singular: azureuserassignedidentity + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AzureUserAssignedIdentity represents a reference to an Azure + access key ID and secret access key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AzureUserAssignedIdentitySpec. + properties: + allowedNamespaces: + description: "AllowedNamespaces is a selector of namespaces that AzureClusters + can use this ClusterPrincipal from. This is a standard Kubernetes + LabelSelector, a label query over a set of resources. The result + of matchLabels and matchExpressions are ANDed. Controllers must + not support AzureClusters in namespaces outside this selector. \n + An empty selector (default) indicates that AzureClusters can use + this AzureUserAssignedIdentity from any namespace. This field is + intentionally not a pointer because the nil behavior (no namespaces) + is undesirable here." + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + name: + type: string + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: tenantID clientID' + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + required: + - name + - secretRef + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 8cf3f4ded60..5eef750fd75 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -8,6 +8,7 @@ resources: - bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml - bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml - bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml + - bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml - bases/exp.infrastructure.cluster.x-k8s.io_azuremachinepools.yaml - bases/exp.infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml - bases/exp.infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml diff --git a/config/kustomization.yaml b/config/kustomization.yaml index 341ea922aa7..6e2c0370b0d 100644 --- a/config/kustomization.yaml +++ b/config/kustomization.yaml @@ -7,6 +7,7 @@ resources: - crd - webhook - default + - aadpodidentity patchesJson6902: - target: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 13a1cfb83b1..20ba28ae21b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -5,6 +5,7 @@ metadata: namespace: system labels: control-plane: capz-controller-manager + aadpodidbinding: capz-controller-aadpodidentity-selector spec: selector: matchLabels: @@ -14,6 +15,7 @@ spec: metadata: labels: control-plane: capz-controller-manager + aadpodidbinding: capz-controller-aadpodidentity-selector spec: containers: - args: @@ -34,4 +36,17 @@ spec: httpGet: path: /healthz port: healthz + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace terminationGracePeriodSeconds: 10 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9a43bdf39e3..fc93ac6f65f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -29,6 +29,24 @@ rules: - patch - update - watch +- apiGroups: + - aadpodidentity.k8s.io + resources: + - azureidentities + - azureidentities/status + verbs: + - get + - list + - watch +- apiGroups: + - aadpodidentity.k8s.io + resources: + - azureidentitybindings + - azureidentitybindings/status + verbs: + - get + - list + - watch - apiGroups: - cluster.x-k8s.io resources: @@ -145,6 +163,19 @@ rules: - get - patch - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - azureclusteridentities + - azureclusteridentities/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - infrastructure.cluster.x-k8s.io resources: diff --git a/controllers/azurecluster_controller.go b/controllers/azurecluster_controller.go index d255ce1927a..c7bb726d38c 100644 --- a/controllers/azurecluster_controller.go +++ b/controllers/azurecluster_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/predicates" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -85,6 +86,7 @@ func (r *AzureClusterReconciler) SetupWithManager(mgr ctrl.Manager, options cont // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azuremachinetemplates;azuremachinetemplates/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=azureclusteridentities;azureclusteridentities/status,verbs=get;list;watch;create;update;patch;delete // Reconcile idempotently gets, creates, and updates a cluster. func (r *AzureClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) { @@ -133,7 +135,7 @@ func (r *AzureClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, ret } // Create the scope. - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: log, Cluster: cluster, @@ -175,6 +177,27 @@ func (r *AzureClusterReconciler) reconcileNormal(ctx context.Context, clusterSco return reconcile.Result{}, err } + if azureCluster.Spec.IdentityRef != nil { + identity, err := GetClusterIdentityFromRef(ctx, clusterScope.Client, azureCluster.Namespace, azureCluster.Spec.IdentityRef) + if err != nil { + return reconcile.Result{}, err + } + if !identity.ClusterNamespaceAllowed(azureCluster.Namespace) { + conditions.MarkFalse(azureCluster, infrav1.NetworkInfrastructureReadyCondition, infrav1.NamespaceNotAllowedByIdentity, clusterv1.ConditionSeverityError, "") + return reconcile.Result{}, errors.New("AzureClusterIdentity list of allowed namespaces doesn't include current cluster namespace") + } + if identity.Namespace == azureCluster.Namespace { + patchhelper, err := patch.NewHelper(identity, r.Client) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "failed to init patch helper") + } + identity.ObjectMeta.OwnerReferences = azureCluster.GetOwnerReferences() + if err := patchhelper.Patch(ctx, identity); err != nil { + return reconcile.Result{}, err + } + } + } + // Handle backcompat for CidrBlock if clusterScope.Vnet().CidrBlock != "" { message := "vnet cidrBlock is deprecated, use cidrBlocks instead" diff --git a/controllers/azurecluster_reconciler_test.go b/controllers/azurecluster_reconciler_test.go index 75044c6dd50..08996bc7aba 100644 --- a/controllers/azurecluster_reconciler_test.go +++ b/controllers/azurecluster_reconciler_test.go @@ -25,6 +25,7 @@ import ( "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api-provider-azure/cloud/mocks" "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" @@ -113,7 +114,9 @@ func TestAzureClusterReconcilerDelete(t *testing.T) { tc.expect(groupsMock.EXPECT(), vnetMock.EXPECT(), sgMock.EXPECT(), rtMock.EXPECT(), subnetsMock.EXPECT(), publicIPMock.EXPECT(), lbMock.EXPECT(), dnsMock.EXPECT()) r := &azureClusterReconciler{ - scope: &scope.ClusterScope{}, + scope: &scope.ClusterScope{ + AzureCluster: &infrav1.AzureCluster{}, + }, groupsSvc: groupsMock, vnetSvc: vnetMock, securityGroupSvc: sgMock, diff --git a/controllers/azureidentity_controller.go b/controllers/azureidentity_controller.go new file mode 100644 index 00000000000..78770d84c2b --- /dev/null +++ b/controllers/azureidentity_controller.go @@ -0,0 +1,166 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "time" + + aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" + "github.com/go-logr/logr" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/api/trace" + "go.opentelemetry.io/otel/label" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/tools/record" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/util/identity" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// AzureIdentityReconciler reconciles azure identity objects +type AzureIdentityReconciler struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + ReconcileTimeout time.Duration +} + +// SetupWithManager initializes this controller with a manager +func (r *AzureIdentityReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { + log := r.Log.WithValues("controller", "AzureIdentity") + c, err := ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&infrav1.AzureCluster{}). + WithEventFilter(predicates.ResourceNotPaused(log)). // don't queue reconcile if resource is paused + Build(r) + if err != nil { + return errors.Wrapf(err, "error creating controller") + } + + // Add a watch on clusterv1.Cluster object for unpause notifications. + if err = c.Watch( + &source.Kind{Type: &clusterv1.Cluster{}}, + &handler.EnqueueRequestsFromMapFunc{ + ToRequests: util.ClusterToInfrastructureMapFunc(infrav1.GroupVersion.WithKind("AzureCluster")), + }, + predicates.ClusterUnpaused(log), + ); err != nil { + return errors.Wrapf(err, "failed adding a watch for ready clusters") + } + + return nil +} + +// +kubebuilder:rbac:groups=aadpodidentity.k8s.io,resources=azureidentities;azureidentities/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=aadpodidentity.k8s.io,resources=azureidentitybindings;azureidentitybindings/status,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch + +// Reconcile reconciles the Azure identity. +func (r *AzureIdentityReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, reterr error) { + ctx, cancel := context.WithTimeout(context.Background(), reconciler.DefaultedLoopTimeout(r.ReconcileTimeout)) + defer cancel() + log := r.Log.WithValues("namespace", req.Namespace, "azureIdentity", req.Name) + + ctx, span := tele.Tracer().Start(ctx, "controllers.AzureIdentityReconciler.Reconcile", + trace.WithAttributes( + label.String("namespace", req.Namespace), + label.String("name", req.Name), + label.String("kind", "AzureCluster"), + )) + defer span.End() + + // Fetch the AzureCluster instance + azureCluster := &infrav1.AzureCluster{} + err := r.Get(ctx, req.NamespacedName, azureCluster) + if err != nil { + if apierrors.IsNotFound(err) { + r.Recorder.Eventf(azureCluster, corev1.EventTypeNormal, "AzureClusterObjectNotFound", err.Error()) + log.Info("object was not found") + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + log = log.WithValues("azurecluster", azureCluster.Name) + + // get all the bindings + var bindings aadpodv1.AzureIdentityBindingList + if err := r.List(ctx, &bindings, client.InNamespace(infrav1.ControllerNamespace)); err != nil { + return ctrl.Result{}, err + } + + bindingsToDelete := []aadpodv1.AzureIdentityBinding{} + for _, b := range bindings.Items { + log = log.WithValues("azureidentitybinding", b.Name) + + binding := b + clusterName := binding.ObjectMeta.Labels[clusterv1.ClusterLabelName] + clusterNamespace := binding.ObjectMeta.Labels[infrav1.ClusterLabelNamespace] + + key := client.ObjectKey{Name: clusterName, Namespace: clusterNamespace} + azCluster := &infrav1.AzureCluster{} + if err := r.Get(ctx, key, azCluster); err != nil { + if apierrors.IsNotFound(err) { + bindingsToDelete = append(bindingsToDelete, b) + continue + } else { + return ctrl.Result{}, errors.Wrapf(err, "failed to get AzureCluster") + } + + } + expectedIdentityName := identity.GetAzureIdentityName(azCluster.Name, azCluster.Namespace, azCluster.Spec.IdentityRef.Name) + if binding.Spec.AzureIdentity != expectedIdentityName { + bindingsToDelete = append(bindingsToDelete, b) + } + } + + // delete bindings and identites no longer used by a cluster + for _, bindingToDelete := range bindingsToDelete { + binding := bindingToDelete + identityName := binding.Spec.AzureIdentity + if err := r.Client.Delete(ctx, &binding); err != nil { + r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity for AzureCluster", err.Error()) + log.Error(err, "failed to delete AzureIdentityBinding") + return ctrl.Result{}, err + } + azureIdentity := &aadpodv1.AzureIdentity{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: identityName, Namespace: infrav1.ControllerNamespace}, azureIdentity); err != nil { + log.Error(err, "failed to fetch AzureIdentity") + return ctrl.Result{}, err + } + if err := r.Client.Delete(ctx, azureIdentity); err != nil { + r.Recorder.Eventf(azureCluster, corev1.EventTypeWarning, "Error reconciling AzureIdentity for AzureCluster", err.Error()) + log.Error(err, "failed to delete AzureIdentity") + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} diff --git a/controllers/azurejson_machine_controller.go b/controllers/azurejson_machine_controller.go index bfa2108e28d..83b7b662927 100644 --- a/controllers/azurejson_machine_controller.go +++ b/controllers/azurejson_machine_controller.go @@ -167,7 +167,7 @@ func (r *AzureJSONMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, } // Create the scope. - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: log, Cluster: cluster, diff --git a/controllers/azurejson_machinepool_controller.go b/controllers/azurejson_machinepool_controller.go index 165899f6e3a..3544167fd21 100644 --- a/controllers/azurejson_machinepool_controller.go +++ b/controllers/azurejson_machinepool_controller.go @@ -125,7 +125,7 @@ func (r *AzureJSONMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Res } // Create the scope. - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: log, Cluster: cluster, diff --git a/controllers/azurejson_machinetemplate_controller.go b/controllers/azurejson_machinetemplate_controller.go index 2bfcd422f55..d1c38ffaa25 100644 --- a/controllers/azurejson_machinetemplate_controller.go +++ b/controllers/azurejson_machinetemplate_controller.go @@ -121,7 +121,7 @@ func (r *AzureJSONTemplateReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result } // Create the scope. - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: log, Cluster: cluster, diff --git a/controllers/azuremachine_controller.go b/controllers/azuremachine_controller.go index 9e76d91e308..4c1b60e7a3f 100644 --- a/controllers/azuremachine_controller.go +++ b/controllers/azuremachine_controller.go @@ -180,7 +180,7 @@ func (r *AzureMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, ret logger = logger.WithValues("AzureCluster", azureCluster.Name) // Create the cluster scope - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: logger, Cluster: cluster, diff --git a/controllers/azuremachine_controller_test.go b/controllers/azuremachine_controller_test.go index 2a1d7eb9ac5..834a6d7b6e1 100644 --- a/controllers/azuremachine_controller_test.go +++ b/controllers/azuremachine_controller_test.go @@ -166,7 +166,7 @@ func TestConditions(t *testing.T) { Log: klogr.New(), } - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.TODO(), scope.ClusterScopeParams{ AzureClients: scope.AzureClients{ Authorizer: autorest.NullAuthorizer{}, }, diff --git a/controllers/helpers.go b/controllers/helpers.go index e4cf6dad3f3..599876e9b7d 100644 --- a/controllers/helpers.go +++ b/controllers/helpers.go @@ -387,3 +387,20 @@ func ShouldDeleteIndividualResources(ctx context.Context, clusterScope *scope.Cl // Instead, take the long way and delete all resources one by one. return err != nil || !managed } + +// GetClusterIdentityFromRef returns the AzureClusterIdentity referenced by the AzureCluster. +func GetClusterIdentityFromRef(ctx context.Context, c client.Client, azureClusterNamespace string, ref *corev1.ObjectReference) (*infrav1.AzureClusterIdentity, error) { + identity := &infrav1.AzureClusterIdentity{} + if ref != nil { + namespace := ref.Namespace + if namespace == "" { + namespace = azureClusterNamespace + } + key := client.ObjectKey{Name: ref.Name, Namespace: namespace} + if err := c.Get(ctx, key, identity); err != nil { + return nil, err + } + return identity, nil + } + return nil, nil +} diff --git a/controllers/helpers_test.go b/controllers/helpers_test.go index d08bbaca003..1e59a9a70a3 100644 --- a/controllers/helpers_test.go +++ b/controllers/helpers_test.go @@ -138,7 +138,7 @@ func TestGetCloudProviderConfig(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.Background(), scope.ClusterScopeParams{ AzureClients: scope.AzureClients{ Authorizer: autorest.NullAuthorizer{}, }, @@ -199,7 +199,7 @@ func TestReconcileAzureSecret(t *testing.T) { cluster.Default() azureCluster.Default() - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(context.Background(), scope.ClusterScopeParams{ AzureClients: scope.AzureClients{ Authorizer: autorest.NullAuthorizer{}, }, diff --git a/docs/book/src/topics/multitenancy.md b/docs/book/src/topics/multitenancy.md new file mode 100644 index 00000000000..ea084f21672 --- /dev/null +++ b/docs/book/src/topics/multitenancy.md @@ -0,0 +1,83 @@ +# Multi-tenancy + +To enable single controller multi-tenancy, a different Identity can be added to the Azure Cluster that will be used as the Azure Identity when creating Azure resources related to that cluster. + +This is achieved using the [aad-pod-identity](https://azure.github.io/aad-pod-identity) library. + +## Service Principal Identity + +Once a new SP Identity is created in Azure, the corresponding values should be used to create an `AzureClusterIdentity` resource: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureClusterIdentity +metadata: + name: example-identity + namespace: default +spec: + type: ServicePrincipal + tenantID: + clientID: + clientSecret: {"name":"","namespace":"default"} + allowedNamespaces: + - + +``` +The password will need to be added in a secret similar to the following example + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: +type: Opaque +data: + clientSecret: +``` + +OR the password can also as a Certificate + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: +type: Opaque +data: + certificate: CERTIFICATE + password: PASSWORD +``` + +## allowedNamespaces +AllowedNamespaces is an array of namespaces that AzureClusters can use this Identity from. CAPZ will not support AzureClusters in namespaces outside this list. +An empty list (default) indicates that AzureCluster can use this AzureClusterIdentity from any namespace. + +## IdentityRef in AzureCluster + +The Identity can be added to an `AzureCluster` by using `IdentityRef` field: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: example-cluster + namespace: default +spec: + location: eastus + networkSpec: + vnet: + name: example-cluster-vnet + resourceGroup: example-cluster + subscriptionID: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureClusterIdentity + name: + namespace: +``` + +For more details on how aad-pod-identity works, please check the guide [here](https://azure.github.io/aad-pod-identity/docs/). + +## User Assiged Identity + +_will be supported in a future release_ \ No newline at end of file diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index bcf2a7e1859..be14d4d299d 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -188,7 +188,7 @@ func (r *AzureMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, logger = logger.WithValues("AzureCluster", azureCluster.Name) // Create the cluster scope - clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + clusterScope, err := scope.NewClusterScope(ctx, scope.ClusterScopeParams{ Client: r.Client, Logger: logger, Cluster: cluster, diff --git a/go.mod b/go.mod index 0c666383cf5..b829af00d35 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,10 @@ module sigs.k8s.io/cluster-api-provider-azure go 1.13 require ( + github.com/Azure/aad-pod-identity v1.6.3 github.com/Azure/azure-sdk-for-go v48.2.0+incompatible github.com/Azure/go-autorest/autorest v0.11.11 + github.com/Azure/go-autorest/autorest/adal v0.9.5 github.com/Azure/go-autorest/autorest/azure/auth v0.5.3 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect diff --git a/go.sum b/go.sum index 3c195a601c6..6d28d1a0159 100644 --- a/go.sum +++ b/go.sum @@ -32,42 +32,58 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/aad-pod-identity v1.6.3 h1:S9ucEc0Fjlj3nLJhEItjxhHySgYzU5i+mer9fo+Fu4M= +github.com/Azure/aad-pod-identity v1.6.3/go.mod h1:wFUg5YGthk9OLfwg0vImAf6i4vsw17xMgQ8j3MbyvrM= +github.com/Azure/azure-sdk-for-go v40.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v48.2.0+incompatible h1:+t2P1j1r5N6lYgPiiz7ZbEVZFkWjVe9WhHbMm0gg8hw= github.com/Azure/azure-sdk-for-go v48.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.9/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.11 h1:k/wzH9pA3hrtFNsEhJ5SqPEs75W3bzS8VOYA/fJ0j1k= github.com/Azure/go-autorest/autorest v0.11.11/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/azure/auth v0.1.0/go.mod h1:Gf7/i2FUpyb/sGBLIFxTBzrNzBo7aPXXE3ZVeDRwdpM= github.com/Azure/go-autorest/autorest/azure/auth v0.5.3 h1:lZifaPRAk1bqg5vGqreL6F8uLC5V0fDpY8nFvc3boFc= github.com/Azure/go-autorest/autorest/azure/auth v0.5.3/go.mod h1:4bJZhUhcq8LB20TruwHbAQsmUs2Xh+QR7utuJpLXX3A= +github.com/Azure/go-autorest/autorest/azure/cli v0.1.0/go.mod h1:Dk8CUAt/b/PzkfeRsWzVG9Yj3ps8mS8ECztu43rdU8U= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0 h1:Ww5g4zThfD/6cLb4z6xxgeyDa7QDkizMkJKe0ysZXp0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -134,6 +150,7 @@ github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -154,6 +171,7 @@ github.com/coredns/corefile-migration v1.0.11/go.mod h1:RMy/mXdeDlYwzt0vdMEJvT2h github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.3.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -304,6 +322,7 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -389,6 +408,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= @@ -405,6 +425,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -607,6 +628,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= @@ -627,6 +649,7 @@ github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= @@ -639,6 +662,7 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= @@ -778,12 +802,15 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -906,6 +933,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1095,6 +1123,7 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1171,18 +1200,21 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= k8s.io/api v0.17.9/go.mod h1:avJJAA1fSV6tnbCGW2K+S+ilDFW7WpNr5BScoiZ1M1U= k8s.io/api v0.17.14 h1:hIsaOvARCNsyc6xBfjEX86eyz8rFjlauSsPBrojUxOg= k8s.io/api v0.17.14/go.mod h1:pjv1T5ozRyCkLvZt6BJL92CZ6WZ1hmI6GcxlU0Oy+q4= k8s.io/apiextensions-apiserver v0.17.9 h1:GWtUr9LErCZBV7QEUIF7wiICPG6wzPukFRrwDv/AIdM= k8s.io/apiextensions-apiserver v0.17.9/go.mod h1:p2C9cDflVAUPMl5/QOMHxnSzQWF/cDqu7AP2KUXHHMA= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.9/go.mod h1:Lg8zZ5iC/O8UjCqW6DNhcQG2m4TdjF9kwG3891OWbbA= k8s.io/apimachinery v0.17.14 h1:PSlsJ6G92PRz/D0uRzMrQ+bKr750NppKYPa8Qg9GF5I= k8s.io/apimachinery v0.17.14/go.mod h1:T54ZSpncArE25c5r2PbUPsLeTpkPWY/ivafigSX6+xk= k8s.io/apiserver v0.17.9 h1:q50QEJ51xdHy2Gl1lo9yJexiyixxof/yDUFdWNnZxh0= k8s.io/apiserver v0.17.9/go.mod h1:Qaxd3EbeoPRBHVMtFyuKNAObqP6VAkzIMyWYz8KuE2k= k8s.io/cli-runtime v0.17.14/go.mod h1:GECO6R4KmPmww6ScOP4WHmCB5R0flukK8Jay4CJHa5k= +k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI= k8s.io/client-go v0.17.9/go.mod h1:3cM92qAd1XknA5IRkRfpJhl9OQjkYy97ZEUio70wVnI= k8s.io/client-go v0.17.14 h1:J+rMxOy+tAmsD4VBjW6CduZfygaUa3AhdxQsfsX8nhc= k8s.io/client-go v0.17.14/go.mod h1:1SFRrpQKPcUNGwKy+M7IH+kAiMhw3DnkSth+XtWh5Tk= diff --git a/main.go b/main.go index 65e16d3bffc..ac66b94f697 100644 --- a/main.go +++ b/main.go @@ -32,7 +32,9 @@ import ( otelProm "go.opentelemetry.io/otel/exporters/metric/prometheus" "go.opentelemetry.io/otel/exporters/trace/jaeger" "go.opentelemetry.io/otel/sdk/trace" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" clientgoscheme "k8s.io/client-go/kubernetes/scheme" cgrecord "k8s.io/client-go/tools/record" "k8s.io/klog" @@ -46,6 +48,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" + aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" infrav1alpha2 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha2" infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" "sigs.k8s.io/cluster-api-provider-azure/controllers" @@ -73,6 +76,18 @@ func init() { _ = clusterv1.AddToScheme(scheme) _ = clusterv1exp.AddToScheme(scheme) // +kubebuilder:scaffold:scheme + + // Add aadpodidentity v1 to the scheme. + aadPodIdentityGroupVersion := schema.GroupVersion{Group: aadpodv1.CRDGroup, Version: aadpodv1.CRDVersion} + scheme.AddKnownTypes(aadPodIdentityGroupVersion, + &aadpodv1.AzureIdentity{}, + &aadpodv1.AzureIdentityList{}, + &aadpodv1.AzureIdentityBinding{}, + &aadpodv1.AzureIdentityBindingList{}, + &aadpodv1.AzurePodIdentityException{}, + &aadpodv1.AzurePodIdentityExceptionList{}, + ) + metav1.AddToGroupVersion(scheme, aadPodIdentityGroupVersion) } var ( @@ -278,6 +293,15 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "AzureJSONMachine") os.Exit(1) } + if err = (&controllers.AzureIdentityReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("AzureIdentity"), + Recorder: mgr.GetEventRecorderFor("azureidentity-reconciler"), + ReconcileTimeout: reconcileTimeout, + }).SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: azureClusterConcurrency}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AzureIdentity") + os.Exit(1) + } // just use CAPI MachinePool feature flag rather than create a new one setupLog.V(1).Info(fmt.Sprintf("%+v\n", feature.Gates)) if feature.Gates.Enabled(capifeature.MachinePool) { diff --git a/templates/cluster-template-multi-tenancy.yaml b/templates/cluster-template-multi-tenancy.yaml new file mode 100644 index 00000000000..42abb714c8a --- /dev/null +++ b/templates/cluster-template-multi-tenancy.yaml @@ -0,0 +1,217 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + namespace: ${CLUSTER_IDENTITY_NAMESPACE} + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureClusterIdentity +metadata: + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + clientID: ${AZURE_CLUSTER_IDENTITY_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal diff --git a/templates/flavors/multi-tenancy/azure-cluster-identity.yaml b/templates/flavors/multi-tenancy/azure-cluster-identity.yaml new file mode 100644 index 00000000000..9a77568c663 --- /dev/null +++ b/templates/flavors/multi-tenancy/azure-cluster-identity.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureClusterIdentity +metadata: + name: "${CLUSTER_IDENTITY_NAME}" +spec: + type: ServicePrincipal + tenantID: "${AZURE_TENANT_ID}" + clientID: "${AZURE_CLUSTER_IDENTITY_CLIENT_ID}" + clientSecret: {"name":"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}","namespace":"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}"} diff --git a/templates/flavors/multi-tenancy/kustomization.yaml b/templates/flavors/multi-tenancy/kustomization.yaml new file mode 100644 index 00000000000..047db53df8b --- /dev/null +++ b/templates/flavors/multi-tenancy/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: default +resources: + - ../default + - azure-cluster-identity.yaml +patchesStrategicMerge: + - patches/azurecluster-identity-ref.yaml diff --git a/templates/flavors/multi-tenancy/patches/azurecluster-identity-ref.yaml b/templates/flavors/multi-tenancy/patches/azurecluster-identity-ref.yaml new file mode 100644 index 00000000000..e2323439139 --- /dev/null +++ b/templates/flavors/multi-tenancy/patches/azurecluster-identity-ref.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureClusterIdentity + name: "${CLUSTER_IDENTITY_NAME}" + namespace: "${CLUSTER_IDENTITY_NAMESPACE}" + diff --git a/templates/test/cluster-template-prow-multi-tenancy.yaml b/templates/test/cluster-template-prow-multi-tenancy.yaml new file mode 100644 index 00000000000..839b1d40090 --- /dev/null +++ b/templates/test/cluster-template-prow-multi-tenancy.yaml @@ -0,0 +1,241 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + namespace: ${CLUSTER_IDENTITY_NAMESPACE} + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureClusterIdentity +metadata: + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + clientID: ${AZURE_CLUSTER_IDENTITY_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: v1 +data: ${CNI_RESOURCES_IPV6} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-multi-tenancy/cni-resource-set.yaml b/templates/test/prow-multi-tenancy/cni-resource-set.yaml new file mode 100644 index 00000000000..de760b6e79c --- /dev/null +++ b/templates/test/prow-multi-tenancy/cni-resource-set.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +data: ${CNI_RESOURCES_IPV6} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-multi-tenancy/kustomization.yaml b/templates/test/prow-multi-tenancy/kustomization.yaml new file mode 100644 index 00000000000..3cfdf179902 --- /dev/null +++ b/templates/test/prow-multi-tenancy/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../flavors/multi-tenancy + - cni-resource-set.yaml +patchesStrategicMerge: + - ../patches/tags.yaml + - patches/cluster-cni.yaml diff --git a/templates/test/prow-multi-tenancy/patches/cluster-cni.yaml b/templates/test/prow-multi-tenancy/patches/cluster-cni.yaml new file mode 100644 index 00000000000..751f46e6154 --- /dev/null +++ b/templates/test/prow-multi-tenancy/patches/cluster-cni.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default + labels: + cni: "${CLUSTER_NAME}-crs-0" diff --git a/test/e2e/azure_identity.go b/test/e2e/azure_identity.go new file mode 100644 index 00000000000..d3c96c67ce3 --- /dev/null +++ b/test/e2e/azure_identity.go @@ -0,0 +1,66 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/cluster-api/test/framework" +) + +// AzureServicePrincipalIdentitySpecInput is the input for AzureServicePrincipalIdentitySpec +type AzureServicePrincipalIdentitySpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string + SkipCleanup bool + IPv6 bool +} + +// AzureServicePrincipalIdentitySpec implements a test that verifies Azure identity can be added to +// an AzureCluster and used as the identity for that cluster. +func AzureServicePrincipalIdentitySpec(ctx context.Context, inputGetter func() AzureServicePrincipalIdentitySpecInput) { + var ( + specName = "azure-identity" + input AzureServicePrincipalIdentitySpecInput + ) + + input = inputGetter() + Expect(input.ClusterName).NotTo(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + By("creating Azure clients with the workload cluster's subscription") + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + authorizer, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + vmsClient := compute.NewVirtualMachinesClient(subscriptionID) + vmsClient.Authorizer = authorizer + + rgName := input.ClusterName + machines, err := vmsClient.List(ctx, rgName) + Expect(err).NotTo(HaveOccurred()) + Expect(len(machines.Values())).To(BeNumerically(">", 0)) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index d1aa63d400c..fa03f54c714 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -28,6 +28,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" @@ -350,4 +351,60 @@ var _ = Describe("Workload cluster creation", func() { }) }) }) + + Context("Creating a cluster using a different SP identity", func() { + BeforeEach(func() { + spClientSecret := os.Getenv("AZURE_MULTI_TENANCY_SECRET") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sp-identity-secret", + Namespace: namespace.Name, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"clientSecret": []byte(spClientSecret)}, + } + err := bootstrapClusterProxy.GetClient().Create(ctx, secret) + Expect(err).ToNot(HaveOccurred()) + }) + + It("with a single control plane node and 1 node", func() { + spClientID := os.Getenv("AZURE_MULTI_TENANCY_ID") + identityName := e2eConfig.GetVariable(MultiTenancyIdentityName) + os.Setenv("CLUSTER_IDENTITY_NAME", identityName) + os.Setenv("CLUSTER_IDENTITY_NAMESPACE", namespace.Name) + os.Setenv("AZURE_CLUSTER_IDENTITY_CLIENT_ID", spClientID) + os.Setenv("AZURE_CLUSTER_IDENTITY_SECRET_NAME", "sp-identity-secret") + os.Setenv("AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE", namespace.Name) + + result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "multi-tenancy", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + cluster = result.Cluster + + Context("Validating identity", func() { + AzureServicePrincipalIdentitySpec(ctx, func() AzureServicePrincipalIdentitySpecInput { + return AzureServicePrincipalIdentitySpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + }) + }) }) diff --git a/test/e2e/common.go b/test/e2e/common.go index 921177e0f74..03cb43860ee 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -40,18 +40,19 @@ import ( // Test suite constants for e2e config variables const ( - RedactLogScriptPath = "REDACT_LOG_SCRIPT" - AzureLocation = "AZURE_LOCATION" - AzureResourceGroup = "AZURE_RESOURCE_GROUP" - AzureVNetName = "AZURE_VNET_NAME" - AzureInternalLBIP = "AZURE_INTERNAL_LB_IP" - AzureCPSubnetCidr = "AZURE_CP_SUBNET_CIDR" - AzureNodeSubnetCidr = "AZURE_NODE_SUBNET_CIDR" - CNIPathIPv6 = "CNI_IPV6" - CNIResourcesIPv6 = "CNI_RESOURCES_IPV6" - VMSSHPort = "VM_SSH_PORT" - JobName = "JOB_NAME" - Timestamp = "TIMESTAMP" + RedactLogScriptPath = "REDACT_LOG_SCRIPT" + AzureLocation = "AZURE_LOCATION" + AzureResourceGroup = "AZURE_RESOURCE_GROUP" + AzureVNetName = "AZURE_VNET_NAME" + AzureInternalLBIP = "AZURE_INTERNAL_LB_IP" + AzureCPSubnetCidr = "AZURE_CP_SUBNET_CIDR" + AzureNodeSubnetCidr = "AZURE_NODE_SUBNET_CIDR" + CNIPathIPv6 = "CNI_IPV6" + CNIResourcesIPv6 = "CNI_RESOURCES_IPV6" + VMSSHPort = "VM_SSH_PORT" + MultiTenancyIdentityName = "MULTI_TENANCY_IDENTITY_NAME" + JobName = "JOB_NAME" + Timestamp = "TIMESTAMP" ) func Byf(format string, a ...interface{}) { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index fffe40acaf2..2bd91834405 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -60,6 +60,8 @@ providers: targetName: "cluster-template-private.yaml" - sourcePath: "${PWD}/templates/test/cluster-template-prow-ci-version.yaml" targetName: "cluster-template-conformance-ci-artifacts.yaml" + - sourcePath: "${PWD}/templates/test/cluster-template-prow-multi-tenancy.yaml" + targetName: "cluster-template-multi-tenancy.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.19.4}" @@ -77,6 +79,7 @@ variables: CONFORMANCE_WORKER_MACHINE_COUNT: "5" CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1" VM_SSH_PORT: "22" + MULTI_TENANCY_IDENTITY_NAME: "multi-tenancy-identity" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 822d7c8303e..eafbf922a43 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -27,11 +27,14 @@ import ( "strings" "testing" + aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" @@ -159,6 +162,19 @@ func initScheme() *runtime.Scheme { scheme := runtime.NewScheme() framework.TryAddDefaultSchemes(scheme) Expect(infrav1.AddToScheme(scheme)).To(Succeed()) + // Add aadpodidentity v1 to the scheme. + aadPodIdentityGroupVersion := schema.GroupVersion{Group: aadpodv1.CRDGroup, Version: aadpodv1.CRDVersion} + scheme.AddKnownTypes(aadPodIdentityGroupVersion, + &aadpodv1.AzureIdentity{}, + &aadpodv1.AzureIdentityList{}, + &aadpodv1.AzureIdentityBinding{}, + &aadpodv1.AzureIdentityBindingList{}, + &aadpodv1.AzureAssignedIdentity{}, + &aadpodv1.AzureAssignedIdentityList{}, + &aadpodv1.AzurePodIdentityException{}, + &aadpodv1.AzurePodIdentityExceptionList{}, + ) + metav1.AddToGroupVersion(scheme, aadPodIdentityGroupVersion) return scheme } diff --git a/util/identity/defaults.go b/util/identity/defaults.go new file mode 100644 index 00000000000..95c2b475d3d --- /dev/null +++ b/util/identity/defaults.go @@ -0,0 +1,24 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package identity + +import "fmt" + +// GetAzureIdentityName formats the name of the AzureIdentity created by the capz controller. +func GetAzureIdentityName(clusterName, clusterNamespace, identityName string) string { + return fmt.Sprintf("%s-%s-%s", clusterName, clusterNamespace, identityName) +}