diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..30cc62af5a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.sh text eol=lf +*.yaml text eol=lf \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 61be31a390..e8c636043f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # Build the manager binary -FROM golang:1.17.3 as toolchain +FROM golang:1.20.2 as toolchain # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org diff --git a/Makefile b/Makefile index 872a76b306..ab3747ccf7 100644 --- a/Makefile +++ b/Makefile @@ -77,11 +77,11 @@ endif # Release variables -STAGING_REGISTRY ?= gcr.io/k8s-staging-cluster-api-aws +STAGING_REGISTRY ?= gcr.io/spectro-dev-public/cluster-api-aws STAGING_BUCKET ?= artifacts.k8s-staging-cluster-api-aws.appspot.com BUCKET ?= $(STAGING_BUCKET) PROD_REGISTRY := registry.k8s.io/cluster-api-aws -REGISTRY ?= $(STAGING_REGISTRY) +REGISTRY ?= gcr.io/spectro-dev-public/amit/cluster-api-aws RELEASE_TAG ?= $(shell git describe --abbrev=0 2>/dev/null) PULL_BASE_REF ?= $(RELEASE_TAG) # PULL_BASE_REF will be provided by Prow RELEASE_ALIAS_TAG ?= $(PULL_BASE_REF) @@ -92,8 +92,9 @@ BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) # image name used to build the cmd/clusterawsadm TOOLCHAIN_IMAGE := toolchain -TAG ?= dev -ARCH ?= $(shell go env GOARCH) +TAG ?= spectro-v1.5.0-$(shell date +%Y%m%d) +#ARCH ?= $(shell go env GOARCH) +ARCH ?= amd64 ALL_ARCH ?= amd64 arm arm64 ppc64le s390x # main controller @@ -343,7 +344,8 @@ clusterawsadm: ## Build clusterawsadm binary .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for controller-manager - docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" . -t $(CORE_CONTROLLER_IMG)-$(ARCH):$(TAG) + docker build --build-arg ARCH=$(ARCH) --build-arg LDFLAGS="$(LDFLAGS)" . -t $(CORE_CONTROLLER_IMG):$(TAG) + @echo $(CORE_CONTROLLER_IMG):$(TAG) .PHONY: docker-build-all ## Build all the architecture docker images docker-build-all: $(addprefix docker-build-,$(ALL_ARCH)) @@ -486,7 +488,7 @@ compiled-manifest: $(RELEASE_DIR) $(KUSTOMIZE) ## Compile the manifest files .PHONY: docker-push docker-push: ## Push the docker image - docker push $(CORE_CONTROLLER_IMG)-$(ARCH):$(TAG) + docker push $(CORE_CONTROLLER_IMG):$(TAG) .PHONY: docker-push-all ## Push all the architecture docker images docker-push-all: $(addprefix docker-push-,$(ALL_ARCH)) diff --git a/OWNERS b/OWNERS index e07729547d..70b9881ebc 100644 --- a/OWNERS +++ b/OWNERS @@ -4,9 +4,11 @@ approvers: - sig-cluster-lifecycle-leads - cluster-api-admins - cluster-api-maintainers + - cluster-api-aws-admins - cluster-api-aws-maintainers reviewers: + - cluster-api-aws-admins - cluster-api-aws-maintainers - cluster-api-aws-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 1639bdb53c..1b268b0c88 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,28 +1,35 @@ # See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md aliases: - # Correct as of 2022/01/05 + # Correct as of 2022/10/13 sig-cluster-lifecycle-leads: - fabriziopandini - justinsb - neolit123 - - vincepri - # Correct as of 2022/01/05 + - timothysc + # Correct as of 2022/10/13 cluster-api-admins: - CecileRobertMichon - vincepri - # Correct as of 2022/01/05 + # Correct as of 2022/10/13 cluster-api-maintainers: - CecileRobertMichon - enxebre - fabriziopandini + - sbueringer - vincepri + # Correct as of 2022/10/19 + cluster-api-aws-admins: + - richardcase + - sedefsavas cluster-api-aws-maintainers: - richardcase - sedefsavas - cluster-api-aws-reviewers: + - Skarlso - Ankitasw - - dthorsen - dlipovetsky + cluster-api-aws-reviewers: + - dthorsen - pydctw - shivi28 + - AverageMarcus diff --git a/api/v1alpha3/awscluster_conversion.go b/api/v1alpha3/awscluster_conversion.go index 91de470914..df2ede9273 100644 --- a/api/v1alpha3/awscluster_conversion.go +++ b/api/v1alpha3/awscluster_conversion.go @@ -17,14 +17,17 @@ limitations under the License. package v1alpha3 import ( + "fmt" "unsafe" apiconversion "k8s.io/apimachinery/pkg/conversion" + + "sigs.k8s.io/controller-runtime/pkg/conversion" + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" clusterv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" - "sigs.k8s.io/controller-runtime/pkg/conversion" ) // ConvertTo converts the v1alpha3 AWSCluster receiver to a v1beta1 AWSCluster. @@ -55,6 +58,25 @@ func (r *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { } dst.Spec.S3Bucket = restored.Spec.S3Bucket + dst.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID = restored.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID + dst.Spec.NetworkSpec.VPC.EnableIPv6 = restored.Spec.NetworkSpec.VPC.EnableIPv6 + dst.Spec.NetworkSpec.VPC.IPv6CidrBlock = restored.Spec.NetworkSpec.VPC.IPv6CidrBlock + dst.Spec.NetworkSpec.VPC.IPv6Pool = restored.Spec.NetworkSpec.VPC.IPv6Pool + + for i := range dst.Spec.NetworkSpec.Subnets { + var found bool + for k := range restored.Spec.NetworkSpec.Subnets { + if dst.Spec.NetworkSpec.Subnets[i].ID == restored.Spec.NetworkSpec.Subnets[k].ID { + dst.Spec.NetworkSpec.Subnets[i].IsIPv6 = restored.Spec.NetworkSpec.Subnets[i].IsIPv6 + dst.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock = restored.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock + found = true + break + } + } + if !found { + return fmt.Errorf("subnet with id %s not found amongts restored subnets", dst.Spec.NetworkSpec.Subnets[i].ID) + } + } return nil } @@ -131,3 +153,15 @@ func Convert_v1beta1_AWSLoadBalancerSpec_To_v1alpha3_AWSLoadBalancerSpec(in *inf func Convert_v1beta1_AWSClusterSpec_To_v1alpha3_AWSClusterSpec(in *infrav1.AWSClusterSpec, out *AWSClusterSpec, s apiconversion.Scope) error { return autoConvert_v1beta1_AWSClusterSpec_To_v1alpha3_AWSClusterSpec(in, out, s) } + +func Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(in *infrav1.VPCSpec, out *VPCSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(in, out, s) +} + +func Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(in *infrav1.SubnetSpec, out *SubnetSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(in, out, s) +} + +func Convert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(in *infrav1.AWSClusterStatus, out *AWSClusterStatus, s apiconversion.Scope) error { + return autoConvert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(in, out, s) +} diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 638fd46bfb..7c628e8556 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -161,11 +161,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.AWSClusterStatus)(nil), (*AWSClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(a.(*v1beta1.AWSClusterStatus), b.(*AWSClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*AWSIdentityReference)(nil), (*v1beta1.AWSIdentityReference)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_AWSIdentityReference_To_v1beta1_AWSIdentityReference(a.(*AWSIdentityReference), b.(*v1beta1.AWSIdentityReference), scope) }); err != nil { @@ -441,21 +436,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.SubnetSpec)(nil), (*SubnetSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(a.(*v1beta1.SubnetSpec), b.(*SubnetSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VPCSpec)(nil), (*v1beta1.VPCSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_VPCSpec_To_v1beta1_VPCSpec(a.(*VPCSpec), b.(*v1beta1.VPCSpec), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.VPCSpec)(nil), (*VPCSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(a.(*v1beta1.VPCSpec), b.(*VPCSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Volume)(nil), (*v1beta1.Volume)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_Volume_To_v1beta1_Volume(a.(*Volume), b.(*v1beta1.Volume), scope) }); err != nil { @@ -491,6 +476,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.AWSClusterStatus)(nil), (*AWSClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(a.(*v1beta1.AWSClusterStatus), b.(*AWSClusterStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AWSLoadBalancerSpec)(nil), (*AWSLoadBalancerSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AWSLoadBalancerSpec_To_v1alpha3_AWSLoadBalancerSpec(a.(*v1beta1.AWSLoadBalancerSpec), b.(*AWSLoadBalancerSpec), scope) }); err != nil { @@ -516,6 +506,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.SubnetSpec)(nil), (*SubnetSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(a.(*v1beta1.SubnetSpec), b.(*SubnetSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.VPCSpec)(nil), (*VPCSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(a.(*v1beta1.VPCSpec), b.(*VPCSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.Volume)(nil), (*Volume)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_Volume_To_v1alpha3_Volume(a.(*v1beta1.Volume), b.(*Volume), scope) }); err != nil { @@ -811,6 +811,7 @@ func autoConvert_v1beta1_AWSClusterSpec_To_v1alpha3_AWSClusterSpec(in *v1beta1.A return err } out.Region = in.Region + // WARNING: in.Partition requires manual conversion: does not exist in peer-type out.SSHKeyName = (*string)(unsafe.Pointer(in.SSHKeyName)) if err := apiv1alpha3.Convert_v1beta1_APIEndpoint_To_v1alpha3_APIEndpoint(&in.ControlPlaneEndpoint, &out.ControlPlaneEndpoint, s); err != nil { return err @@ -833,6 +834,7 @@ func autoConvert_v1beta1_AWSClusterSpec_To_v1alpha3_AWSClusterSpec(in *v1beta1.A } out.IdentityRef = (*AWSIdentityReference)(unsafe.Pointer(in.IdentityRef)) // WARNING: in.S3Bucket requires manual conversion: does not exist in peer-type + // WARNING: in.AssociateOIDCProvider requires manual conversion: does not exist in peer-type return nil } @@ -1004,14 +1006,10 @@ func autoConvert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(in *v1bet } else { out.Conditions = nil } + // WARNING: in.OIDCProvider requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus is an autogenerated conversion function. -func Convert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(in *v1beta1.AWSClusterStatus, out *AWSClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta1_AWSClusterStatus_To_v1alpha3_AWSClusterStatus(in, out, s) -} - func autoConvert_v1alpha3_AWSIdentityReference_To_v1beta1_AWSIdentityReference(in *AWSIdentityReference, out *v1beta1.AWSIdentityReference, s conversion.Scope) error { out.Name = in.Name out.Kind = v1beta1.AWSIdentityKind(in.Kind) @@ -1905,7 +1903,17 @@ func autoConvert_v1alpha3_NetworkSpec_To_v1beta1_NetworkSpec(in *NetworkSpec, ou if err := Convert_v1alpha3_VPCSpec_To_v1beta1_VPCSpec(&in.VPC, &out.VPC, s); err != nil { return err } - out.Subnets = *(*v1beta1.Subnets)(unsafe.Pointer(&in.Subnets)) + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make(v1beta1.Subnets, len(*in)) + for i := range *in { + if err := Convert_v1alpha3_SubnetSpec_To_v1beta1_SubnetSpec(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Subnets = nil + } out.CNI = (*v1beta1.CNISpec)(unsafe.Pointer(in.CNI)) out.SecurityGroupOverrides = *(*map[v1beta1.SecurityGroupRole]string)(unsafe.Pointer(&in.SecurityGroupOverrides)) return nil @@ -1920,7 +1928,17 @@ func autoConvert_v1beta1_NetworkSpec_To_v1alpha3_NetworkSpec(in *v1beta1.Network if err := Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(&in.VPC, &out.VPC, s); err != nil { return err } - out.Subnets = *(*Subnets)(unsafe.Pointer(&in.Subnets)) + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make(Subnets, len(*in)) + for i := range *in { + if err := Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Subnets = nil + } out.CNI = (*CNISpec)(unsafe.Pointer(in.CNI)) out.SecurityGroupOverrides = *(*map[SecurityGroupRole]string)(unsafe.Pointer(&in.SecurityGroupOverrides)) return nil @@ -2016,19 +2034,16 @@ func Convert_v1alpha3_SubnetSpec_To_v1beta1_SubnetSpec(in *SubnetSpec, out *v1be func autoConvert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(in *v1beta1.SubnetSpec, out *SubnetSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock + // WARNING: in.IPv6CidrBlock requires manual conversion: does not exist in peer-type out.AvailabilityZone = in.AvailabilityZone out.IsPublic = in.IsPublic + // WARNING: in.IsIPv6 requires manual conversion: does not exist in peer-type out.RouteTableID = (*string)(unsafe.Pointer(in.RouteTableID)) out.NatGatewayID = (*string)(unsafe.Pointer(in.NatGatewayID)) out.Tags = *(*Tags)(unsafe.Pointer(&in.Tags)) return nil } -// Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec is an autogenerated conversion function. -func Convert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(in *v1beta1.SubnetSpec, out *SubnetSpec, s conversion.Scope) error { - return autoConvert_v1beta1_SubnetSpec_To_v1alpha3_SubnetSpec(in, out, s) -} - func autoConvert_v1alpha3_VPCSpec_To_v1beta1_VPCSpec(in *VPCSpec, out *v1beta1.VPCSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock @@ -2047,18 +2062,17 @@ func Convert_v1alpha3_VPCSpec_To_v1beta1_VPCSpec(in *VPCSpec, out *v1beta1.VPCSp func autoConvert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(in *v1beta1.VPCSpec, out *VPCSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock + // WARNING: in.EnableIPv6 requires manual conversion: does not exist in peer-type + // WARNING: in.IPv6CidrBlock requires manual conversion: does not exist in peer-type + // WARNING: in.IPv6Pool requires manual conversion: does not exist in peer-type out.InternetGatewayID = (*string)(unsafe.Pointer(in.InternetGatewayID)) + // WARNING: in.EgressOnlyInternetGatewayID requires manual conversion: does not exist in peer-type out.Tags = *(*Tags)(unsafe.Pointer(&in.Tags)) out.AvailabilityZoneUsageLimit = (*int)(unsafe.Pointer(in.AvailabilityZoneUsageLimit)) out.AvailabilityZoneSelection = (*AZSelectionScheme)(unsafe.Pointer(in.AvailabilityZoneSelection)) return nil } -// Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec is an autogenerated conversion function. -func Convert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(in *v1beta1.VPCSpec, out *VPCSpec, s conversion.Scope) error { - return autoConvert_v1beta1_VPCSpec_To_v1alpha3_VPCSpec(in, out, s) -} - func autoConvert_v1alpha3_Volume_To_v1beta1_Volume(in *Volume, out *v1beta1.Volume, s conversion.Scope) error { out.DeviceName = in.DeviceName out.Size = in.Size diff --git a/api/v1alpha4/awscluster_conversion.go b/api/v1alpha4/awscluster_conversion.go index 56bea69ced..221ef5b1c8 100644 --- a/api/v1alpha4/awscluster_conversion.go +++ b/api/v1alpha4/awscluster_conversion.go @@ -17,12 +17,15 @@ limitations under the License. package v1alpha4 import ( + "fmt" + apiconversion "k8s.io/apimachinery/pkg/conversion" - infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" clusterv1alpha4 "sigs.k8s.io/cluster-api/api/v1alpha4" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" ) // ConvertTo converts the v1alpha4 AWSCluster receiver to a v1beta1 AWSCluster. @@ -46,6 +49,25 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { } dst.Spec.S3Bucket = restored.Spec.S3Bucket + dst.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID = restored.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID + dst.Spec.NetworkSpec.VPC.EnableIPv6 = restored.Spec.NetworkSpec.VPC.EnableIPv6 + dst.Spec.NetworkSpec.VPC.IPv6CidrBlock = restored.Spec.NetworkSpec.VPC.IPv6CidrBlock + dst.Spec.NetworkSpec.VPC.IPv6Pool = restored.Spec.NetworkSpec.VPC.IPv6Pool + + for i := range dst.Spec.NetworkSpec.Subnets { + var found bool + for k := range restored.Spec.NetworkSpec.Subnets { + if dst.Spec.NetworkSpec.Subnets[i].ID == restored.Spec.NetworkSpec.Subnets[k].ID { + dst.Spec.NetworkSpec.Subnets[i].IsIPv6 = restored.Spec.NetworkSpec.Subnets[i].IsIPv6 + dst.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock = restored.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock + found = true + break + } + } + if !found { + return fmt.Errorf("subnet with id %s not found amongts restored subnets", dst.Spec.NetworkSpec.Subnets[i].ID) + } + } return nil } @@ -100,3 +122,10 @@ func (r *AWSClusterList) ConvertFrom(srcRaw conversion.Hub) error { func Convert_v1beta1_AWSLoadBalancerSpec_To_v1alpha4_AWSLoadBalancerSpec(in *infrav1.AWSLoadBalancerSpec, out *AWSLoadBalancerSpec, s apiconversion.Scope) error { return autoConvert_v1beta1_AWSLoadBalancerSpec_To_v1alpha4_AWSLoadBalancerSpec(in, out, s) } + +func Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in *infrav1.SubnetSpec, out *SubnetSpec, s apiconversion.Scope) error { + return autoConvert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in, out, s) +} +func Convert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(in *infrav1.AWSClusterStatus, out *AWSClusterStatus, s apiconversion.Scope) error { + return autoConvert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(in, out, s) +} diff --git a/api/v1alpha4/conversion.go b/api/v1alpha4/conversion.go index 6d8c4cfc44..b33428b39d 100644 --- a/api/v1alpha4/conversion.go +++ b/api/v1alpha4/conversion.go @@ -18,9 +18,15 @@ package v1alpha4 import ( "k8s.io/apimachinery/pkg/conversion" + "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" ) func Convert_v1beta1_AWSClusterSpec_To_v1alpha4_AWSClusterSpec(in *v1beta1.AWSClusterSpec, out *AWSClusterSpec, s conversion.Scope) error { return autoConvert_v1beta1_AWSClusterSpec_To_v1alpha4_AWSClusterSpec(in, out, s) } + +func Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(in *v1beta1.VPCSpec, out *VPCSpec, s conversion.Scope) error { + // Discard EgressOnlyInternetGatewayID + return autoConvert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(in, out, s) +} diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index 0b0fd48f87..ecc44bbac9 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -180,11 +180,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.AWSClusterStatus)(nil), (*AWSClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(a.(*v1beta1.AWSClusterStatus), b.(*AWSClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*AWSClusterTemplate)(nil), (*v1beta1.AWSClusterTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_AWSClusterTemplate_To_v1beta1_AWSClusterTemplate(a.(*AWSClusterTemplate), b.(*v1beta1.AWSClusterTemplate), scope) }); err != nil { @@ -510,7 +505,7 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.SubnetSpec)(nil), (*SubnetSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + if err := s.AddConversionFunc((*v1beta1.SubnetSpec)(nil), (*SubnetSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(a.(*v1beta1.SubnetSpec), b.(*SubnetSpec), scope) }); err != nil { return err @@ -520,11 +515,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.VPCSpec)(nil), (*VPCSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(a.(*v1beta1.VPCSpec), b.(*VPCSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*Volume)(nil), (*v1beta1.Volume)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_Volume_To_v1beta1_Volume(a.(*Volume), b.(*v1beta1.Volume), scope) }); err != nil { @@ -540,6 +530,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.AWSClusterStatus)(nil), (*AWSClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(a.(*v1beta1.AWSClusterStatus), b.(*AWSClusterStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AWSClusterTemplateResource)(nil), (*AWSClusterTemplateResource)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AWSClusterTemplateResource_To_v1alpha4_AWSClusterTemplateResource(a.(*v1beta1.AWSClusterTemplateResource), b.(*AWSClusterTemplateResource), scope) }); err != nil { @@ -560,6 +555,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.AWSMachineTemplate)(nil), (*AWSMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AWSMachineTemplate_To_v1alpha4_AWSMachineTemplate(a.(*v1beta1.AWSMachineTemplate), b.(*AWSMachineTemplate), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta1.VPCSpec)(nil), (*VPCSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(a.(*v1beta1.VPCSpec), b.(*VPCSpec), scope) + }); err != nil { + return err + } return nil } @@ -872,6 +877,7 @@ func autoConvert_v1beta1_AWSClusterSpec_To_v1alpha4_AWSClusterSpec(in *v1beta1.A return err } out.Region = in.Region + // WARNING: in.Partition requires manual conversion: does not exist in peer-type out.SSHKeyName = (*string)(unsafe.Pointer(in.SSHKeyName)) if err := apiv1alpha4.Convert_v1beta1_APIEndpoint_To_v1alpha4_APIEndpoint(&in.ControlPlaneEndpoint, &out.ControlPlaneEndpoint, s); err != nil { return err @@ -894,6 +900,7 @@ func autoConvert_v1beta1_AWSClusterSpec_To_v1alpha4_AWSClusterSpec(in *v1beta1.A } out.IdentityRef = (*AWSIdentityReference)(unsafe.Pointer(in.IdentityRef)) // WARNING: in.S3Bucket requires manual conversion: does not exist in peer-type + // WARNING: in.AssociateOIDCProvider requires manual conversion: does not exist in peer-type return nil } @@ -1055,14 +1062,10 @@ func autoConvert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(in *v1bet } else { out.Conditions = nil } + // WARNING: in.OIDCProvider requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus is an autogenerated conversion function. -func Convert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(in *v1beta1.AWSClusterStatus, out *AWSClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta1_AWSClusterStatus_To_v1alpha4_AWSClusterStatus(in, out, s) -} - func autoConvert_v1alpha4_AWSClusterTemplate_To_v1beta1_AWSClusterTemplate(in *AWSClusterTemplate, out *v1beta1.AWSClusterTemplate, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha4_AWSClusterTemplateSpec_To_v1beta1_AWSClusterTemplateSpec(&in.Spec, &out.Spec, s); err != nil { @@ -2002,7 +2005,17 @@ func autoConvert_v1alpha4_NetworkSpec_To_v1beta1_NetworkSpec(in *NetworkSpec, ou if err := Convert_v1alpha4_VPCSpec_To_v1beta1_VPCSpec(&in.VPC, &out.VPC, s); err != nil { return err } - out.Subnets = *(*v1beta1.Subnets)(unsafe.Pointer(&in.Subnets)) + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make(v1beta1.Subnets, len(*in)) + for i := range *in { + if err := Convert_v1alpha4_SubnetSpec_To_v1beta1_SubnetSpec(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Subnets = nil + } out.CNI = (*v1beta1.CNISpec)(unsafe.Pointer(in.CNI)) out.SecurityGroupOverrides = *(*map[v1beta1.SecurityGroupRole]string)(unsafe.Pointer(&in.SecurityGroupOverrides)) return nil @@ -2017,7 +2030,17 @@ func autoConvert_v1beta1_NetworkSpec_To_v1alpha4_NetworkSpec(in *v1beta1.Network if err := Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(&in.VPC, &out.VPC, s); err != nil { return err } - out.Subnets = *(*Subnets)(unsafe.Pointer(&in.Subnets)) + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make(Subnets, len(*in)) + for i := range *in { + if err := Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Subnets = nil + } out.CNI = (*CNISpec)(unsafe.Pointer(in.CNI)) out.SecurityGroupOverrides = *(*map[SecurityGroupRole]string)(unsafe.Pointer(&in.SecurityGroupOverrides)) return nil @@ -2139,19 +2162,16 @@ func Convert_v1alpha4_SubnetSpec_To_v1beta1_SubnetSpec(in *SubnetSpec, out *v1be func autoConvert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in *v1beta1.SubnetSpec, out *SubnetSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock + // WARNING: in.IPv6CidrBlock requires manual conversion: does not exist in peer-type out.AvailabilityZone = in.AvailabilityZone out.IsPublic = in.IsPublic + // WARNING: in.IsIPv6 requires manual conversion: does not exist in peer-type out.RouteTableID = (*string)(unsafe.Pointer(in.RouteTableID)) out.NatGatewayID = (*string)(unsafe.Pointer(in.NatGatewayID)) out.Tags = *(*Tags)(unsafe.Pointer(&in.Tags)) return nil } -// Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec is an autogenerated conversion function. -func Convert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in *v1beta1.SubnetSpec, out *SubnetSpec, s conversion.Scope) error { - return autoConvert_v1beta1_SubnetSpec_To_v1alpha4_SubnetSpec(in, out, s) -} - func autoConvert_v1alpha4_VPCSpec_To_v1beta1_VPCSpec(in *VPCSpec, out *v1beta1.VPCSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock @@ -2170,18 +2190,17 @@ func Convert_v1alpha4_VPCSpec_To_v1beta1_VPCSpec(in *VPCSpec, out *v1beta1.VPCSp func autoConvert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(in *v1beta1.VPCSpec, out *VPCSpec, s conversion.Scope) error { out.ID = in.ID out.CidrBlock = in.CidrBlock + // WARNING: in.EnableIPv6 requires manual conversion: does not exist in peer-type + // WARNING: in.IPv6CidrBlock requires manual conversion: does not exist in peer-type + // WARNING: in.IPv6Pool requires manual conversion: does not exist in peer-type out.InternetGatewayID = (*string)(unsafe.Pointer(in.InternetGatewayID)) + // WARNING: in.EgressOnlyInternetGatewayID requires manual conversion: does not exist in peer-type out.Tags = *(*Tags)(unsafe.Pointer(&in.Tags)) out.AvailabilityZoneUsageLimit = (*int)(unsafe.Pointer(in.AvailabilityZoneUsageLimit)) out.AvailabilityZoneSelection = (*AZSelectionScheme)(unsafe.Pointer(in.AvailabilityZoneSelection)) return nil } -// Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec is an autogenerated conversion function. -func Convert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(in *v1beta1.VPCSpec, out *VPCSpec, s conversion.Scope) error { - return autoConvert_v1beta1_VPCSpec_To_v1alpha4_VPCSpec(in, out, s) -} - func autoConvert_v1alpha4_Volume_To_v1beta1_Volume(in *Volume, out *v1beta1.Volume, s conversion.Scope) error { out.DeviceName = in.DeviceName out.Size = in.Size diff --git a/api/v1beta1/awscluster_types.go b/api/v1beta1/awscluster_types.go index bae8e6ecc6..d9ef29631b 100644 --- a/api/v1beta1/awscluster_types.go +++ b/api/v1beta1/awscluster_types.go @@ -39,6 +39,10 @@ type AWSClusterSpec struct { // The AWS Region the cluster lives in. Region string `json:"region,omitempty"` + // Partition is the AWS security partition being used. Defaults to "aws" + // +optional + Partition string `json:"partition,omitempty"` + // SSHKeyName is the name of the ssh key to attach to the bastion host. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name) // +optional SSHKeyName *string `json:"sshKeyName,omitempty"` @@ -92,11 +96,18 @@ type AWSClusterSpec struct { IdentityRef *AWSIdentityReference `json:"identityRef,omitempty"` // S3Bucket contains options to configure a supporting S3 bucket for this - // cluster - currently used for nodes requiring Ignition + // cluster - Used for nodes requiring Ignition // (https://coreos.github.io/ignition/) for bootstrapping (requires - // BootstrapFormatIgnition feature flag to be enabled). + // BootstrapFormatIgnition feature flag to be enabled) and for storing OIDC endpoint + // certificates for use with IRSA // +optional S3Bucket *S3Bucket `json:"s3Bucket,omitempty"` + + // AssociateOIDCProvider can be enabled to automatically create an identity + // provider and install the pod identity webhook from AWS for use with IRSA. + // This will only work if the S3Bucket is configured properly. + // +kubebuilder:default=false + AssociateOIDCProvider bool `json:"associateOIDCProvider,omitempty"` } // AWSIdentityKind defines allowed AWS identity types. @@ -203,6 +214,10 @@ type AWSClusterStatus struct { FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` Bastion *Instance `json:"bastion,omitempty"` Conditions clusterv1.Conditions `json:"conditions,omitempty"` + + // OIDCProvider holds the status of the identity provider for this cluster + // +optional + OIDCProvider OIDCProviderStatus `json:"oidcProvider,omitempty"` } type S3Bucket struct { diff --git a/api/v1beta1/awsmachine_webhook.go b/api/v1beta1/awsmachine_webhook.go index d393a0ff45..237f657c92 100644 --- a/api/v1beta1/awsmachine_webhook.go +++ b/api/v1beta1/awsmachine_webhook.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + "fmt" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -114,7 +115,8 @@ func (r *AWSMachine) ValidateUpdate(old runtime.Object) error { } if !cmp.Equal(oldAWSMachineSpec, newAWSMachineSpec) { - allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "cannot be modified")) + s := fmt.Sprintf("oldAWSMachineSpec: %s, newAWSMachineSpec: %s do not match", oldAWSMachineSpec, newAWSMachineSpec) + allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "cannot be modified."+s)) } return aggregateObjErrors(r.GroupVersionKind().GroupKind(), r.Name, allErrs) @@ -237,6 +239,10 @@ func (r *AWSMachine) Default() { r.Spec.CloudInit.SecureSecretsBackend = SecretBackendSecretsManager } + if r.Spec.RootVolume != nil && !r.ignitionEnabled() { + r.Spec.RootVolume.DeviceName = "" + } + if r.ignitionEnabled() && r.Spec.Ignition.Version == "" { if r.Spec.Ignition == nil { r.Spec.Ignition = &Ignition{} diff --git a/api/v1beta1/awsmachine_webhook_test.go b/api/v1beta1/awsmachine_webhook_test.go index d16917cdff..0d699009a7 100644 --- a/api/v1beta1/awsmachine_webhook_test.go +++ b/api/v1beta1/awsmachine_webhook_test.go @@ -43,6 +43,18 @@ func TestAWSMachine_Create(t *testing.T) { machine *AWSMachine wantErr bool }{ + { + name: "ensure root volume with device name is overwritten on create", + machine: &AWSMachine{ + Spec: AWSMachineSpec{ + RootVolume: &Volume{ + DeviceName: "devicename", + Size: 10, + }, + }, + }, + wantErr: false, + }, { name: "ensure IOPS exists if type equal to io1", machine: &AWSMachine{ @@ -354,6 +366,29 @@ func TestAWSMachine_Update(t *testing.T) { }, wantErr: true, }, + { + name: "change in root volume device name", + oldMachine: &AWSMachine{ + Spec: AWSMachineSpec{ + ProviderID: nil, + AdditionalTags: nil, + AdditionalSecurityGroups: nil, + RootVolume: &Volume{ + Size: 60, + }, + }, + }, + newMachine: &AWSMachine{ + Spec: AWSMachineSpec{ + ProviderID: pointer.StringPtr("ID"), + RootVolume: &Volume{ + DeviceName: "rootdevicename", + Size: 60, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { ctx := context.TODO() diff --git a/api/v1beta1/conditions_consts.go b/api/v1beta1/conditions_consts.go index 8c972f026e..0850a6d5ec 100644 --- a/api/v1beta1/conditions_consts.go +++ b/api/v1beta1/conditions_consts.go @@ -61,6 +61,14 @@ const ( InternetGatewayFailedReason = "InternetGatewayFailed" ) +const ( + // EgressOnlyInternetGatewayReadyCondition reports on the successful reconciliation of egress only internet gateways. + // Only applicable to managed clusters. + EgressOnlyInternetGatewayReadyCondition clusterv1.ConditionType = "EgressOnlyInternetGatewayReady" + // EgressOnlyInternetGatewayFailedReason used when errors occur during egress only internet gateway reconciliation. + EgressOnlyInternetGatewayFailedReason = "EgressOnlyInternetGatewayFailed" +) + const ( // NatGatewaysReadyCondition reports successful reconciliation of NAT gateways. // Only applicable to managed clusters. diff --git a/api/v1beta1/network_types.go b/api/v1beta1/network_types.go index adc70f2200..d104253f88 100644 --- a/api/v1beta1/network_types.go +++ b/api/v1beta1/network_types.go @@ -174,10 +174,28 @@ type VPCSpec struct { // Defaults to 10.0.0.0/16. CidrBlock string `json:"cidrBlock,omitempty"` + // EnableIPv6 requests an Amazon-provided IPv6 CIDR block with a /56 prefix length for + // the VPC. You cannot specify the range of IP addresses, or the size of the + // CIDR block. + // +optional + EnableIPv6 bool `json:"enableIPv6"` + + // IPv6CidrBlock is the CIDR block provided by Amazon when VPC has enabled IPv6. + // +optional + IPv6CidrBlock string `json:"ipv6CidrBlock,omitempty"` + + // IPv6Pool is the IP pool which must be defined in case of BYO IP is defined. + // +optional + IPv6Pool string `json:"ipv6Pool,omitempty"` + // InternetGatewayID is the id of the internet gateway associated with the VPC. // +optional InternetGatewayID *string `json:"internetGatewayId,omitempty"` + // EgressOnlyInternetGatewayID is the id of the egress only internet gateway associated with an IPv6 enabled VPC. + // +optional + EgressOnlyInternetGatewayID *string `json:"egressOnlyInternetGatewayId,omitempty"` + // Tags is a collection of tags describing the resource. Tags Tags `json:"tags,omitempty"` @@ -222,6 +240,10 @@ type SubnetSpec struct { // CidrBlock is the CIDR block to be used when the provider creates a managed VPC. CidrBlock string `json:"cidrBlock,omitempty"` + // IPv6CidrBlock is the IPv6 CIDR block to be used when the provider creates a managed VPC. + // A subnet can have an IPv4 and an IPv6 address. + IPv6CidrBlock string `json:"ipv6CidrBlock,omitempty"` + // AvailabilityZone defines the availability zone to use for this subnet in the cluster's region. AvailabilityZone string `json:"availabilityZone,omitempty"` @@ -229,6 +251,10 @@ type SubnetSpec struct { // +optional IsPublic bool `json:"isPublic"` + // IsIPv6 defines the subnet as an IPv6 subnet. A subnet is IPv6 when it is associated with a VPC that has IPv6 enabled. + // +optional + IsIPv6 bool `json:"isIpv6"` + // RouteTableID is the routing table id associated with the subnet. // +optional RouteTableID *string `json:"routeTableId,omitempty"` @@ -285,7 +311,7 @@ func (s Subnets) FindByID(id string) *SubnetSpec { // or if they are in the same vpc and the cidr block is the same. func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec { for _, x := range s { - if (spec.ID != "" && x.ID == spec.ID) || (spec.CidrBlock == x.CidrBlock) { + if (spec.ID != "" && x.ID == spec.ID) || (spec.CidrBlock == x.CidrBlock) || (spec.IPv6CidrBlock != "" && spec.IPv6CidrBlock == x.IPv6CidrBlock) { return &x } } diff --git a/api/v1beta1/s3bucket.go b/api/v1beta1/s3bucket.go index 2e84c6533d..619cf27679 100644 --- a/api/v1beta1/s3bucket.go +++ b/api/v1beta1/s3bucket.go @@ -21,8 +21,6 @@ import ( "net" "k8s.io/apimachinery/pkg/util/validation/field" - - "sigs.k8s.io/cluster-api-provider-aws/feature" ) // Validate validates S3Bucket fields. @@ -37,12 +35,6 @@ func (b *S3Bucket) Validate() []*field.Error { errs = append(errs, field.Required(field.NewPath("spec", "s3Bucket", "name"), "can't be empty")) } - // Feature gate is not enabled but ignition is enabled then send a forbidden error. - if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) { - errs = append(errs, field.Forbidden(field.NewPath("spec", "s3Bucket"), - "can be set only if the BootstrapFormatIgnition feature gate is enabled")) - } - if b.ControlPlaneIAMInstanceProfile == "" { errs = append(errs, field.Required(field.NewPath("spec", "s3Bucket", "controlPlaneIAMInstanceProfiles"), "can't be empty")) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index fe6510380b..a2596451a0 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -291,3 +291,11 @@ const ( // AmazonLinuxGPU is the AmazonLinux GPU AMI type. AmazonLinuxGPU EKSAMILookupType = "AmazonLinuxGPU" ) + +// OIDCProviderStatus holds the status of the AWS OIDC identity provider. +type OIDCProviderStatus struct { + // ARN holds the ARN of the provider + ARN string `json:"arn,omitempty"` + // TrustPolicy contains the boilerplate IAM trust policy to use for IRSA + TrustPolicy string `json:"trustPolicy,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 28901c6380..41b51afe53 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -427,6 +427,7 @@ func (in *AWSClusterStatus) DeepCopyInto(out *AWSClusterStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.OIDCProvider = in.OIDCProvider } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSClusterStatus. @@ -1385,6 +1386,21 @@ func (in *NetworkStatus) DeepCopy() *NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OIDCProviderStatus) DeepCopyInto(out *OIDCProviderStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OIDCProviderStatus. +func (in *OIDCProviderStatus) DeepCopy() *OIDCProviderStatus { + if in == nil { + return nil + } + out := new(OIDCProviderStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RouteTable) DeepCopyInto(out *RouteTable) { *out = *in @@ -1551,6 +1567,11 @@ func (in *VPCSpec) DeepCopyInto(out *VPCSpec) { *out = new(string) **out = **in } + if in.EgressOnlyInternetGatewayID != nil { + in, out := &in.EgressOnlyInternetGatewayID, &out.EgressOnlyInternetGatewayID + *out = new(string) + **out = **in + } if in.Tags != nil { in, out := &in.Tags, &out.Tags *out = make(Tags, len(*in)) diff --git a/bootstrap/eks/api/v1alpha3/conversion.go b/bootstrap/eks/api/v1alpha3/conversion.go index 614e42faf9..c517d9b31a 100644 --- a/bootstrap/eks/api/v1alpha3/conversion.go +++ b/bootstrap/eks/api/v1alpha3/conversion.go @@ -54,6 +54,7 @@ func restoreSpec(rSpec, dSpec *v1beta1.EKSConfigSpec) { } } dSpec.UseMaxPods = rSpec.UseMaxPods + dSpec.ServiceIPV6Cidr = rSpec.ServiceIPV6Cidr } // ConvertFrom converts the v1beta1 EKSConfig receiver to a v1alpha3 EKSConfig. diff --git a/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go b/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go index b748fd9f62..2284c60d1b 100644 --- a/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go +++ b/bootstrap/eks/api/v1alpha3/zz_generated.conversion.go @@ -213,6 +213,7 @@ func autoConvert_v1beta1_EKSConfigSpec_To_v1alpha3_EKSConfigSpec(in *v1beta1.EKS // WARNING: in.APIRetryAttempts requires manual conversion: does not exist in peer-type // WARNING: in.PauseContainer requires manual conversion: does not exist in peer-type // WARNING: in.UseMaxPods requires manual conversion: does not exist in peer-type + // WARNING: in.ServiceIPV6Cidr requires manual conversion: does not exist in peer-type return nil } diff --git a/bootstrap/eks/api/v1alpha4/conversion.go b/bootstrap/eks/api/v1alpha4/conversion.go index 85ccf1ad0c..a6209cec9b 100644 --- a/bootstrap/eks/api/v1alpha4/conversion.go +++ b/bootstrap/eks/api/v1alpha4/conversion.go @@ -54,6 +54,7 @@ func restoreSpec(rSpec, dSpec *v1beta1.EKSConfigSpec) { } } dSpec.UseMaxPods = rSpec.UseMaxPods + dSpec.ServiceIPV6Cidr = rSpec.ServiceIPV6Cidr } // ConvertFrom converts the v1beta1 EKSConfig receiver to a v1alpha4 EKSConfig. diff --git a/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go b/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go index fdee27fe79..17c346bf7c 100644 --- a/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go +++ b/bootstrap/eks/api/v1alpha4/zz_generated.conversion.go @@ -213,6 +213,7 @@ func autoConvert_v1beta1_EKSConfigSpec_To_v1alpha4_EKSConfigSpec(in *v1beta1.EKS // WARNING: in.APIRetryAttempts requires manual conversion: does not exist in peer-type // WARNING: in.PauseContainer requires manual conversion: does not exist in peer-type // WARNING: in.UseMaxPods requires manual conversion: does not exist in peer-type + // WARNING: in.ServiceIPV6Cidr requires manual conversion: does not exist in peer-type return nil } diff --git a/bootstrap/eks/api/v1beta1/eksconfig_types.go b/bootstrap/eks/api/v1beta1/eksconfig_types.go index 5d7be87ed3..259af4c7ac 100644 --- a/bootstrap/eks/api/v1beta1/eksconfig_types.go +++ b/bootstrap/eks/api/v1beta1/eksconfig_types.go @@ -47,11 +47,10 @@ type EKSConfigSpec struct { // +optional UseMaxPods *bool `json:"useMaxPods,omitempty"` - // TODO(richardcase): this can be uncommented when we get to the ipv6/dual-stack implementation // ServiceIPV6Cidr is the ipv6 cidr range of the cluster. If this is specified then // the ip family will be set to ipv6. // +optional - // ServiceIPV6Cidr *string `json:"serviceIPV6Cidr,omitempty"` + ServiceIPV6Cidr *string `json:"serviceIPV6Cidr,omitempty"` } // PauseContainer contains details of pause container. diff --git a/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go index 5125b17e2d..b8342c9a9e 100644 --- a/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go +++ b/bootstrap/eks/api/v1beta1/zz_generated.deepcopy.go @@ -125,6 +125,11 @@ func (in *EKSConfigSpec) DeepCopyInto(out *EKSConfigSpec) { *out = new(bool) **out = **in } + if in.ServiceIPV6Cidr != nil { + in, out := &in.ServiceIPV6Cidr, &out.ServiceIPV6Cidr + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EKSConfigSpec. diff --git a/bootstrap/eks/controllers/eksconfig_controller.go b/bootstrap/eks/controllers/eksconfig_controller.go index 837dcf7824..74caa0bc2d 100644 --- a/bootstrap/eks/controllers/eksconfig_controller.go +++ b/bootstrap/eks/controllers/eksconfig_controller.go @@ -200,11 +200,20 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1 nodeInput.PauseContainerAccount = &config.Spec.PauseContainer.AccountNumber nodeInput.PauseContainerVersion = &config.Spec.PauseContainer.Version } - // TODO(richardcase): uncomment when we support ipv6 / dual stack - /*if config.Spec.ServiceIPV6Cidr != nil && *config.Spec.ServiceIPV6Cidr != "" { + + // Check if IPv6 was provided to the user configuration first + // If not, we also check if the cluster is ipv6 based. + if config.Spec.ServiceIPV6Cidr != nil && *config.Spec.ServiceIPV6Cidr != "" { nodeInput.ServiceIPV6Cidr = config.Spec.ServiceIPV6Cidr nodeInput.IPFamily = pointer.String("ipv6") - }*/ + } + + // we don't want to override any manually set configuration options. + if config.Spec.ServiceIPV6Cidr == nil && controlPlane.Spec.NetworkSpec.VPC.EnableIPv6 { + log.Info("Adding ipv6 data to userdata....") + nodeInput.ServiceIPV6Cidr = pointer.String(controlPlane.Spec.NetworkSpec.VPC.IPv6CidrBlock) + nodeInput.IPFamily = pointer.String("ipv6") + } // generate userdata userDataScript, err := userdata.NewNode(nodeInput) diff --git a/cmd/clusterawsadm/api/bootstrap/v1beta1/defaults.go b/cmd/clusterawsadm/api/bootstrap/v1beta1/defaults.go index f4ed1f2ae5..41d1f6d944 100644 --- a/cmd/clusterawsadm/api/bootstrap/v1beta1/defaults.go +++ b/cmd/clusterawsadm/api/bootstrap/v1beta1/defaults.go @@ -31,6 +31,8 @@ const ( DefaultStackName = "cluster-api-provider-aws-sigs-k8s-io" // DefaultPartitionName is the default security partition for AWS ARNs. DefaultPartitionName = "aws" + // PartitionNameUSGov is the default security partition for AWS ARNs. + PartitionNameUSGov = "aws-us-gov" // DefaultKMSAliasPattern is the default KMS alias. DefaultKMSAliasPattern = "cluster-api-provider-aws-*" // DefaultS3BucketPrefix is the default S3 bucket prefix. diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_control_plane.go b/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_control_plane.go index 012ffe3434..fa42b5307b 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_control_plane.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_control_plane.go @@ -42,6 +42,7 @@ func (t Template) cloudProviderControlPlaneAwsPolicy() *iamv1.PolicyDocument { "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeLaunchConfigurations", "autoscaling:DescribeTags", + "ec2:AssignIpv6Addresses", "ec2:DescribeInstances", "ec2:DescribeImages", "ec2:DescribeRegions", diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_node.go b/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_node.go index 866ad16c86..7d780aac74 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_node.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/cloud_provider_integration_node.go @@ -43,8 +43,13 @@ func (t Template) cloudProviderNodeAwsPolicy() *iamv1.PolicyDocument { Effect: iamv1.EffectAllow, Resource: iamv1.Resources{iamv1.Any}, Action: iamv1.Actions{ + "ec2:AssignIpv6Addresses", "ec2:DescribeInstances", "ec2:DescribeRegions", + "ec2:CreateTags", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeInstanceTypes", "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go index 32f690f4d9..f2405d7fbf 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go @@ -81,12 +81,18 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { Effect: iamv1.EffectAllow, Resource: iamv1.Resources{iamv1.Any}, Action: iamv1.Actions{ + "ec2:AttachNetworkInterface", + "ec2:DetachNetworkInterface", "ec2:AllocateAddress", + "ec2:AssignIpv6Addresses", + "ec2:AssignPrivateIpAddresses", + "ec2:UnassignPrivateIpAddresses", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AuthorizeSecurityGroupIngress", "ec2:CreateInternetGateway", "ec2:CreateNatGateway", + "ec2:CreateNetworkInterface", "ec2:CreateRoute", "ec2:CreateRouteTable", "ec2:CreateSecurityGroup", @@ -107,6 +113,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { "ec2:DescribeAvailabilityZones", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", + "ec2:DescribeInstanceTypes", "ec2:DescribeImages", "ec2:DescribeNatGateways", "ec2:DescribeNetworkInterfaces", @@ -117,6 +124,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument { "ec2:DescribeVpcs", "ec2:DescribeVpcAttribute", "ec2:DescribeVolumes", + "ec2:DescribeTags", "ec2:DetachInternetGateway", "ec2:DisassociateRouteTable", "ec2:DisassociateAddress", @@ -331,7 +339,7 @@ func (t Template) ControllersPolicyEKS() *iamv1.PolicyDocument { "iam:CreateServiceLinkedRole", }, Resource: iamv1.Resources{ - "arn:aws:iam::*:role/aws-service-role/eks-fargate-pods.amazonaws.com/AWSServiceRoleForAmazonEKSForFargate", + "arn:*:iam::*:role/aws-service-role/eks-fargate-pods.amazonaws.com/AWSServiceRoleForAmazonEKSForFargate", }, Condition: iamv1.Conditions{ iamv1.StringLike: map[string]string{"iam:AWSServiceName": "eks-fargate.amazonaws.com"}, diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fargate.go b/cmd/clusterawsadm/cloudformation/bootstrap/fargate.go index 8ba8dfad6c..d0658307d2 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fargate.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fargate.go @@ -19,10 +19,15 @@ package bootstrap import ( bootstrapv1 "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/bootstrap/v1beta1" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks" + "strings" ) -func fargateProfilePolicies(roleSpec *bootstrapv1.AWSIAMRoleSpec) []string { - policies := eks.FargateRolePolicies() +func (t Template) fargateProfilePolicies(roleSpec *bootstrapv1.AWSIAMRoleSpec) []string { + var policies []string + policies = eks.FargateRolePolicies() + if strings.Contains(t.Spec.Partition, bootstrapv1.PartitionNameUSGov) { + policies = eks.FargateRolePoliciesUSGov() + } if roleSpec.ExtraPolicyAttachments != nil { policies = append(policies, roleSpec.ExtraPolicyAttachments...) } diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml index e3cad88330..5183794054 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml index b041414bbd..cf6cb8db69 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml index e32b5ce55a..6340782896 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -139,12 +145,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -165,6 +177,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -175,6 +188,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml index b166a2e6a9..5bf177e324 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml @@ -31,6 +31,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -96,8 +97,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -138,12 +144,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -164,6 +176,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -174,6 +187,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml index 50bc6e5734..d8ebe3f5a0 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml @@ -31,6 +31,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -96,8 +97,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -138,12 +144,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -164,6 +176,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -174,6 +187,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml index 2ac55f69d9..2ae30023bf 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml index 63e7661ad7..1bae5115d4 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml index d5537275af..ba1360cc93 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml index bc1b06bb39..3c602ba831 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml index 0ffbc39727..9997efa211 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml index a6df099478..e64f15b6bd 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml @@ -31,6 +31,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -96,8 +97,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -138,12 +144,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -164,6 +176,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -174,6 +187,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml index 9ab6f126d0..61a9ce477f 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml index aa87fc1cc9..143aa576fa 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml +++ b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml @@ -28,6 +28,7 @@ Resources: - autoscaling:DescribeAutoScalingGroups - autoscaling:DescribeLaunchConfigurations - autoscaling:DescribeTags + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeImages - ec2:DescribeRegions @@ -93,8 +94,13 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AssignIpv6Addresses - ec2:DescribeInstances - ec2:DescribeRegions + - ec2:CreateTags + - ec2:DescribeTags + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstanceTypes - ecr:GetAuthorizationToken - ecr:BatchCheckLayerAvailability - ecr:GetDownloadUrlForLayer @@ -133,12 +139,18 @@ Resources: PolicyDocument: Statement: - Action: + - ec2:AttachNetworkInterface + - ec2:DetachNetworkInterface - ec2:AllocateAddress + - ec2:AssignIpv6Addresses + - ec2:AssignPrivateIpAddresses + - ec2:UnassignPrivateIpAddresses - ec2:AssociateRouteTable - ec2:AttachInternetGateway - ec2:AuthorizeSecurityGroupIngress - ec2:CreateInternetGateway - ec2:CreateNatGateway + - ec2:CreateNetworkInterface - ec2:CreateRoute - ec2:CreateRouteTable - ec2:CreateSecurityGroup @@ -159,6 +171,7 @@ Resources: - ec2:DescribeAvailabilityZones - ec2:DescribeInstances - ec2:DescribeInternetGateways + - ec2:DescribeInstanceTypes - ec2:DescribeImages - ec2:DescribeNatGateways - ec2:DescribeNetworkInterfaces @@ -169,6 +182,7 @@ Resources: - ec2:DescribeVpcs - ec2:DescribeVpcAttribute - ec2:DescribeVolumes + - ec2:DescribeTags - ec2:DetachInternetGateway - ec2:DisassociateRouteTable - ec2:DisassociateAddress diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go b/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go index e96f4b50ac..e991248c34 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/managed_nodegroup.go @@ -16,10 +16,20 @@ limitations under the License. package bootstrap -import "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks" +import ( + bootstrapv1 "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/bootstrap/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/eks" + "strings" +) func (t Template) eksMachinePoolPolicies() []string { - policies := eks.NodegroupRolePolicies() + + var policies []string + + policies = eks.NodegroupRolePolicies() + if strings.Contains(t.Spec.Partition, bootstrapv1.PartitionNameUSGov) { + policies = eks.NodegroupRolePoliciesUSGov() + } if t.Spec.EKS.ManagedMachinePool.ExtraPolicyAttachments != nil { policies = append(policies, t.Spec.EKS.ManagedMachinePool.ExtraPolicyAttachments...) } diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/template.go b/cmd/clusterawsadm/cloudformation/bootstrap/template.go index 2ab20e87d4..ceecfe6904 100644 --- a/cmd/clusterawsadm/cloudformation/bootstrap/template.go +++ b/cmd/clusterawsadm/cloudformation/bootstrap/template.go @@ -200,7 +200,7 @@ func (t Template) RenderCloudFormation() *cloudformation.Template { template.Resources[AWSIAMRoleEKSFargate] = &cfn_iam.Role{ RoleName: expinfrav1.DefaultEKSFargateRole, AssumeRolePolicyDocument: AssumeRolePolicy(iamv1.PrincipalService, []string{eksiam.EKSFargateService}), - ManagedPolicyArns: fargateProfilePolicies(t.Spec.EKS.Fargate), + ManagedPolicyArns: t.fargateProfilePolicies(t.Spec.EKS.Fargate), Tags: converters.MapToCloudFormationTags(t.Spec.EKS.Fargate.Tags), } } diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml index 239714544f..d10500cfea 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigs.yaml @@ -302,6 +302,10 @@ spec: - accountNumber - version type: object + serviceIPV6Cidr: + description: ServiceIPV6Cidr is the ipv6 cidr range of the cluster. + If this is specified then the ip family will be set to ipv6. + type: string useMaxPods: description: UseMaxPods sets --max-pods for the kubelet when true. type: boolean diff --git a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml index 345bdc072d..552693da78 100644 --- a/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml +++ b/config/crd/bases/bootstrap.cluster.x-k8s.io_eksconfigtemplates.yaml @@ -168,6 +168,11 @@ spec: - accountNumber - version type: object + serviceIPV6Cidr: + description: ServiceIPV6Cidr is the ipv6 cidr range of the + cluster. If this is specified then the ip family will be + set to ipv6. + type: string useMaxPods: description: UseMaxPods sets --max-pods for the kubelet when true. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 96e504a2a9..1e8bbb6f7f 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -2175,6 +2175,122 @@ spec: description: AWSManagedControlPlaneSpec defines the desired state of an Amazon EKS Cluster. properties: + VpcCni: + description: VpcCni is used to set configuration options for the VPC + CNI plugin + properties: + env: + description: Env defines a list of environment variables to apply + to the `aws-node` DaemonSet + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + type: object additionalTags: additionalProperties: type: string @@ -2522,6 +2638,16 @@ spec: description: ID defines a unique identifier to reference this resource. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the IPv6 CIDR block to be + used when the provider creates a managed VPC. A subnet + can have an IPv4 and an IPv6 address. + type: string + isIpv6: + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. + type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. A subnet is public when it is associated with a route @@ -2575,6 +2701,16 @@ spec: description: CidrBlock is the CIDR block to be used when the provider creates a managed VPC. Defaults to 10.0.0.0/16. type: string + egressOnlyInternetGatewayId: + description: EgressOnlyInternetGatewayID is the id of the + egress only internet gateway associated with an IPv6 enabled + VPC. + type: string + enableIPv6: + description: EnableIPv6 requests an Amazon-provided IPv6 CIDR + block with a /56 prefix length for the VPC. You cannot specify + the range of IP addresses, or the size of the CIDR block. + type: boolean id: description: ID is the vpc-id of the VPC this provider should use to create resources. @@ -2583,6 +2719,14 @@ spec: description: InternetGatewayID is the id of the internet gateway associated with the VPC. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the CIDR block provided by Amazon + when VPC has enabled IPv6. + type: string + ipv6Pool: + description: IPv6Pool is the IP pool which must be defined + in case of BYO IP is defined. + type: string tags: additionalProperties: type: string @@ -2655,6 +2799,10 @@ spec: prefixing. type: string type: object + partition: + description: Partition is the AWS security partition being used. Defaults + to "aws" + type: string region: description: The AWS Region the cluster lives in. type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index 08c3e27d93..95fdd58af1 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1583,6 +1583,13 @@ spec: resources managed by the AWS provider, in addition to the ones added by default. type: object + associateOIDCProvider: + default: false + description: AssociateOIDCProvider can be enabled to automatically + create an identity provider and install the pod identity webhook + from AWS for use with IRSA. This will only work if the S3Bucket + is configured properly. + type: boolean bastion: description: Bastion contains options to configure the bastion host. properties: @@ -1784,6 +1791,16 @@ spec: description: ID defines a unique identifier to reference this resource. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the IPv6 CIDR block to be + used when the provider creates a managed VPC. A subnet + can have an IPv4 and an IPv6 address. + type: string + isIpv6: + description: IsIPv6 defines the subnet as an IPv6 subnet. + A subnet is IPv6 when it is associated with a VPC that + has IPv6 enabled. + type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. A subnet is public when it is associated with a route @@ -1837,6 +1854,16 @@ spec: description: CidrBlock is the CIDR block to be used when the provider creates a managed VPC. Defaults to 10.0.0.0/16. type: string + egressOnlyInternetGatewayId: + description: EgressOnlyInternetGatewayID is the id of the + egress only internet gateway associated with an IPv6 enabled + VPC. + type: string + enableIPv6: + description: EnableIPv6 requests an Amazon-provided IPv6 CIDR + block with a /56 prefix length for the VPC. You cannot specify + the range of IP addresses, or the size of the CIDR block. + type: boolean id: description: ID is the vpc-id of the VPC this provider should use to create resources. @@ -1845,6 +1872,14 @@ spec: description: InternetGatewayID is the id of the internet gateway associated with the VPC. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the CIDR block provided by Amazon + when VPC has enabled IPv6. + type: string + ipv6Pool: + description: IPv6Pool is the IP pool which must be defined + in case of BYO IP is defined. + type: string tags: additionalProperties: type: string @@ -1852,14 +1887,19 @@ spec: type: object type: object type: object + partition: + description: Partition is the AWS security partition being used. Defaults + to "aws" + type: string region: description: The AWS Region the cluster lives in. type: string s3Bucket: description: S3Bucket contains options to configure a supporting S3 - bucket for this cluster - currently used for nodes requiring Ignition - (https://coreos.github.io/ignition/) for bootstrapping (requires - BootstrapFormatIgnition feature flag to be enabled). + bucket for this cluster - Used for nodes requiring Ignition (https://coreos.github.io/ignition/) + for bootstrapping (requires BootstrapFormatIgnition feature flag + to be enabled) and for storing OIDC endpoint certificates for use + with IRSA properties: controlPlaneIAMInstanceProfile: description: ControlPlaneIAMInstanceProfile is a name of the IAMInstanceProfile, @@ -2329,6 +2369,18 @@ spec: security group to its unique name, if any. type: object type: object + oidcProvider: + description: OIDCProvider holds the status of the identity provider + for this cluster + properties: + arn: + description: ARN holds the ARN of the provider + type: string + trustPolicy: + description: TrustPolicy contains the boilerplate IAM trust policy + to use for IRSA + type: string + type: object ready: default: false type: boolean diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml index 4372bb7db2..2dfb2703d1 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusterstaticidentities.yaml @@ -112,11 +112,11 @@ spec: Optional' properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index b5d35f4a4b..b28c274bd7 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -402,6 +402,13 @@ spec: add to AWS resources managed by the AWS provider, in addition to the ones added by default. type: object + associateOIDCProvider: + default: false + description: AssociateOIDCProvider can be enabled to automatically + create an identity provider and install the pod identity + webhook from AWS for use with IRSA. This will only work + if the S3Bucket is configured properly. + type: boolean bastion: description: Bastion contains options to configure the bastion host. @@ -614,6 +621,16 @@ spec: description: ID defines a unique identifier to reference this resource. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the IPv6 CIDR block + to be used when the provider creates a managed + VPC. A subnet can have an IPv4 and an IPv6 address. + type: string + isIpv6: + description: IsIPv6 defines the subnet as an IPv6 + subnet. A subnet is IPv6 when it is associated + with a VPC that has IPv6 enabled. + type: boolean isPublic: description: IsPublic defines the subnet as a public subnet. A subnet is public when it is associated @@ -671,6 +688,17 @@ spec: when the provider creates a managed VPC. Defaults to 10.0.0.0/16. type: string + egressOnlyInternetGatewayId: + description: EgressOnlyInternetGatewayID is the id + of the egress only internet gateway associated with + an IPv6 enabled VPC. + type: string + enableIPv6: + description: EnableIPv6 requests an Amazon-provided + IPv6 CIDR block with a /56 prefix length for the + VPC. You cannot specify the range of IP addresses, + or the size of the CIDR block. + type: boolean id: description: ID is the vpc-id of the VPC this provider should use to create resources. @@ -679,6 +707,14 @@ spec: description: InternetGatewayID is the id of the internet gateway associated with the VPC. type: string + ipv6CidrBlock: + description: IPv6CidrBlock is the CIDR block provided + by Amazon when VPC has enabled IPv6. + type: string + ipv6Pool: + description: IPv6Pool is the IP pool which must be + defined in case of BYO IP is defined. + type: string tags: additionalProperties: type: string @@ -687,14 +723,19 @@ spec: type: object type: object type: object + partition: + description: Partition is the AWS security partition being + used. Defaults to "aws" + type: string region: description: The AWS Region the cluster lives in. type: string s3Bucket: description: S3Bucket contains options to configure a supporting - S3 bucket for this cluster - currently used for nodes requiring - Ignition (https://coreos.github.io/ignition/) for bootstrapping - (requires BootstrapFormatIgnition feature flag to be enabled). + S3 bucket for this cluster - Used for nodes requiring Ignition + (https://coreos.github.io/ignition/) for bootstrapping (requires + BootstrapFormatIgnition feature flag to be enabled) and + for storing OIDC endpoint certificates for use with IRSA properties: controlPlaneIAMInstanceProfile: description: ControlPlaneIAMInstanceProfile is a name diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml index 899144384e..1a27d2ad26 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmanagedmachinepools.yaml @@ -553,6 +553,165 @@ spec: items: type: string type: array + awsLaunchTemplate: + description: AWSLaunchTemplate specifies the launch template to use + to create the managed node group. If AWSLaunchTemplate is specified, + certain node group configuraions outside of launch template are + prohibited (https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instances. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID or filters. Only one of ID or Filters may + be specified. Specifying more than one will result in a validation + error. + properties: + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the + rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource. + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look + up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + iamInstanceProfile: + description: The name or the Amazon Resource Name (ARN) of the + instance profile associated with the IAM role for the instance. + The instance profile contains the IAM role. + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look + up the image for this machine It will be ignored if an explicit + AMI is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use + for image lookup if AMI is not set. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + name: + description: The name of the launch template. + type: string + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions are options for configuring AWSMachinePool + instances to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to + the instance. Valid values are empty string (do not use SSH + keys), a valid SSH key name, or omitted (use the default SSH + key name) + type: string + versionNumber: + description: 'VersionNumber is the version of the launch template + that is applied. Typically a new version is created when at + least one of the following happens: 1) A new launch template + spec is applied. 2) One or more parameters in an existing template + is changed. 3) A new AMI is discovered.' + format: int64 + type: integer + type: object capacityType: default: onDemand description: CapacityType specifies the capacity type for the ASG diff --git a/controllers/awscluster_controller.go b/controllers/awscluster_controller.go index e943d339d0..c406bbbc33 100644 --- a/controllers/awscluster_controller.go +++ b/controllers/awscluster_controller.go @@ -22,6 +22,8 @@ import ( "net" "time" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/iam" + "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" @@ -73,6 +75,7 @@ type AWSClusterReconciler struct { networkServiceFactory func(scope.ClusterScope) services.NetworkInterface elbServiceFactory func(scope.ELBScope) services.ELBInterface securityGroupFactory func(scope.ClusterScope) services.SecurityGroupInterface + iamServiceFactory func(scope.EC2Scope) services.IAMInterface Endpoints []scope.ServiceEndpoint WatchFilterValue string ExternalResourceGC bool @@ -122,6 +125,14 @@ func (r *AWSClusterReconciler) getSecurityGroupService(scope scope.ClusterScope) return securitygroup.NewService(&scope, securityGroupRolesForCluster(scope)) } +// getIAMService factory func is added for testing purpose so that we can inject mocked EC2Service to the AWSClusterReconciler. +func (r *AWSClusterReconciler) getIAMService(scope scope.EC2Scope) services.IAMInterface { + if r.iamServiceFactory != nil { + return r.iamServiceFactory(scope) + } + return iam.NewService(scope) +} + // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status,verbs=get;list;watch @@ -202,7 +213,7 @@ func (r *AWSClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // Handle non-deleted clusters - return r.reconcileNormal(clusterScope) + return r.reconcileNormal(ctx, clusterScope) } func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, clusterScope *scope.ClusterScope) (reconcile.Result, error) { @@ -213,6 +224,7 @@ func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, clusterScope networkSvc := r.getNetworkService(*clusterScope) sgService := r.getSecurityGroupService(*clusterScope) s3Service := s3.NewService(clusterScope) + iamService := r.getIAMService(clusterScope) if feature.Gates.Enabled(feature.EventBridgeInstanceState) { instancestateSvc := instancestate.NewService(clusterScope) @@ -249,6 +261,10 @@ func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, clusterScope return reconcile.Result{}, err } + if err := iamService.DeleteOIDCProvider(ctx); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "error deleting OIDC Provider") + } + if err := s3Service.DeleteBucket(); err != nil { return reconcile.Result{}, errors.Wrapf(err, "error deleting S3 Bucket") } @@ -259,7 +275,7 @@ func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, clusterScope return reconcile.Result{}, nil } -func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope) (reconcile.Result, error) { +func (r *AWSClusterReconciler) reconcileNormal(ctx context.Context, clusterScope *scope.ClusterScope) (reconcile.Result, error) { clusterScope.Info("Reconciling AWSCluster") awsCluster := clusterScope.AWSCluster @@ -276,6 +292,7 @@ func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope) networkSvc := r.getNetworkService(*clusterScope) sgService := r.getSecurityGroupService(*clusterScope) s3Service := s3.NewService(clusterScope) + iamService := r.getIAMService(clusterScope) if err := networkSvc.ReconcileNetwork(); err != nil { clusterScope.Error(err, "failed to reconcile network") @@ -318,6 +335,11 @@ func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope) return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile S3 Bucket for AWSCluster %s/%s", awsCluster.Namespace, awsCluster.Name) } + if err := iamService.ReconcileOIDCProvider(ctx); err != nil { + clusterScope.Error(err, "failed to reconcile OIDC provider") + return reconcile.Result{RequeueAfter: 15 * time.Second}, nil + } + if awsCluster.Status.Network.APIServerELB.DNSName == "" { conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForDNSNameReason, clusterv1.ConditionSeverityInfo, "") clusterScope.Info("Waiting on API server ELB DNS name") diff --git a/controllers/awsmachine_controller.go b/controllers/awsmachine_controller.go index 8d521148f5..aad20ca383 100644 --- a/controllers/awsmachine_controller.go +++ b/controllers/awsmachine_controller.go @@ -21,6 +21,8 @@ import ( "encoding/json" "fmt" + "k8s.io/apimachinery/pkg/types" + "github.com/aws/aws-sdk-go/aws" ignTypes "github.com/flatcar-linux/ignition/config/v2_3/types" "github.com/go-logr/logr" @@ -428,7 +430,7 @@ func (r *AWSMachineReconciler) findInstance(scope *scope.MachineScope, ec2svc se return instance, nil } -func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope, objectStoreScope scope.S3Scope) (ctrl.Result, error) { +func (r *AWSMachineReconciler) reconcileNormal(ctx context.Context, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope, objectStoreScope scope.S3Scope) (ctrl.Result, error) { machineScope.Info("Reconciling AWSMachine") // If the AWSMachine is in an error state, return early. @@ -560,43 +562,100 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope * // tasks that can take place during all known instance states if machineScope.InstanceIsInKnownState() { - _, err = r.ensureTags(ec2svc, machineScope.AWSMachine, machineScope.GetInstanceID(), machineScope.AdditionalTags()) - if err != nil { - machineScope.Error(err, "failed to ensure tags") + machineScope.V(5).Info("running known state tasks", "instance", instance) + if err := r.knownStateTasks(ctx, machineScope, ec2svc, elbScope, instance); err != nil { return ctrl.Result{}, err } + } - if instance != nil { - r.ensureStorageTags(ec2svc, instance, machineScope.AWSMachine) - } - - if err := r.reconcileLBAttachment(machineScope, elbScope, instance); err != nil { - machineScope.Error(err, "failed to reconcile LB attachment") + // tasks that can only take place during operational instance states + if machineScope.InstanceIsOperational() { + machineScope.V(5).Info("running operational tasks", "instance", instance) + if err := r.operationalTasks(ctx, machineScope, ec2svc, elbScope, instance); err != nil { return ctrl.Result{}, err } } - // tasks that can only take place during operational instance states - if machineScope.InstanceIsOperational() { - machineScope.SetAddresses(instance.Addresses) + return ctrl.Result{}, nil +} + +func (r *AWSMachineReconciler) knownStateTasks(_ context.Context, machineScope *scope.MachineScope, ec2svc services.EC2Interface, elbScope scope.ELBScope, instance *infrav1.Instance) error { + _, err := r.ensureTags(ec2svc, machineScope.AWSMachine, machineScope.GetInstanceID(), machineScope.AdditionalTags()) + if err != nil { + machineScope.Error(err, "failed to ensure tags") + return err + } + + if instance != nil { + r.ensureStorageTags(ec2svc, instance, machineScope.AWSMachine) + } + + if err := r.reconcileLBAttachment(machineScope, elbScope, instance); err != nil { + machineScope.Error(err, "failed to reconcile LB attachment") + return err + } + + return nil +} + +func (r *AWSMachineReconciler) operationalTasks(ctx context.Context, machineScope *scope.MachineScope, ec2svc services.EC2Interface, _ scope.ELBScope, instance *infrav1.Instance) error { + machineScope.SetAddresses(instance.Addresses) - existingSecurityGroups, err := ec2svc.GetInstanceSecurityGroups(*machineScope.GetInstanceID()) + existingSecurityGroups, err := ec2svc.GetInstanceSecurityGroups(*machineScope.GetInstanceID()) + if err != nil { + machineScope.Error(err, "unable to get instance security groups") + return err + } + + // Ensure that the security groups are correct. + _, err = r.ensureSecurityGroups(ec2svc, machineScope, machineScope.AWSMachine.Spec.AdditionalSecurityGroups, existingSecurityGroups) + if err != nil { + conditions.MarkFalse(machineScope.AWSMachine, infrav1.SecurityGroupsReadyCondition, infrav1.SecurityGroupsFailedReason, clusterv1.ConditionSeverityError, err.Error()) + machineScope.Error(err, "unable to ensure security groups") + return err + } + conditions.MarkTrue(machineScope.AWSMachine, infrav1.SecurityGroupsReadyCondition) + + // check if the remote kubeconfig works and annotate the cluster + _, ok := machineScope.InfraCluster.InfraCluster().GetAnnotations()[scope.KubeconfigReadyAnnotation] + if !ok && machineScope.IsControlPlane() { + // if a control plane node is operational check for a kubeconfig and a working control plane node + // and set the annotation so any reconciliation which requires workload api access can complete + remoteClient, err := machineScope.InfraCluster.RemoteClient() if err != nil { - machineScope.Error(err, "unable to get instance security groups") - return ctrl.Result{}, err + return err } - // Ensure that the security groups are correct. - _, err = r.ensureSecurityGroups(ec2svc, machineScope, machineScope.AWSMachine.Spec.AdditionalSecurityGroups, existingSecurityGroups) - if err != nil { - conditions.MarkFalse(machineScope.AWSMachine, infrav1.SecurityGroupsReadyCondition, infrav1.SecurityGroupsFailedReason, clusterv1.ConditionSeverityError, err.Error()) - machineScope.Error(err, "unable to ensure security groups") - return ctrl.Result{}, err + var nodes corev1.NodeList + if err := remoteClient.List(ctx, &nodes, client.MatchingLabels(map[string]string{"node-role.kubernetes.io/control-plane": ""})); err != nil { + return err + } + + oneReady := false + for i := range nodes.Items { + if util.IsNodeReady(&nodes.Items[i]) { + oneReady = true // if one control plane is ready return true + } + } + + if oneReady { + awsCluster := &infrav1.AWSCluster{} + key := types.NamespacedName{Namespace: machineScope.InfraCluster.Namespace(), Name: machineScope.InfraCluster.Name()} + if err := r.Client.Get(ctx, key, awsCluster); err != nil { + return err + } + anno := awsCluster.GetAnnotations() + anno[scope.KubeconfigReadyAnnotation] = "true" + awsCluster.SetAnnotations(anno) + if err := r.Client.Update(ctx, awsCluster); err != nil { + return err + } + } else { + r.Log.Info("waiting for a control plane node to be ready before annotating the cluster, do you need to deploy a CNI?") } - conditions.MarkTrue(machineScope.AWSMachine, infrav1.SecurityGroupsReadyCondition) } - return ctrl.Result{}, nil + return nil } func (r *AWSMachineReconciler) deleteEncryptedBootstrapDataSecret(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper) error { @@ -705,7 +764,7 @@ func (r *AWSMachineReconciler) ignitionUserData(scope *scope.MachineScope, objec return nil, errors.New("object store service not available") } - objectURL, err := objectStoreSvc.Create(scope, userData) + objectURL, err := objectStoreSvc.Create(scope.GetBootstrapDataKey(), userData) if err != nil { return nil, errors.Wrap(err, "creating userdata object") } @@ -734,7 +793,7 @@ func (r *AWSMachineReconciler) ignitionUserData(scope *scope.MachineScope, objec func (r *AWSMachineReconciler) deleteBootstrapData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, objectStoreScope scope.S3Scope) error { if !machineScope.AWSMachine.Spec.CloudInit.InsecureSkipSecretsManager { - if err := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); err != nil { + if err := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); err != nil && !apierrors.IsNotFound(err) { return err } } @@ -772,7 +831,7 @@ func (r *AWSMachineReconciler) deleteIgnitionBootstrapDataFromS3(machineScope *s return nil } - if err := objectStoreSvc.Delete(machineScope); err != nil { + if err := objectStoreSvc.Delete(machineScope.GetBootstrapDataKey()); err != nil { return errors.Wrap(err, "deleting bootstrap data object") } @@ -984,27 +1043,28 @@ func (r *AWSMachineReconciler) indexAWSMachineByInstanceID(o client.Object) []st } func (r *AWSMachineReconciler) ensureStorageTags(ec2svc services.EC2Interface, instance *infrav1.Instance, machine *infrav1.AWSMachine) { - annotations, err := r.machineAnnotationJSON(machine, VolumeTagsLastAppliedAnnotation) + machineAnnotations, err := r.machineAnnotationJSON(machine, VolumeTagsLastAppliedAnnotation) if err != nil { - r.Log.Error(err, "Failed to fetch the annotations for volume tags") + + r.Log.Error(err, "Failed to fetch the machineAnnotations for volume tags") } for _, volumeID := range instance.VolumeIDs { - if subAnnotation, ok := annotations[volumeID].(map[string]interface{}); ok { + if subAnnotation, ok := machineAnnotations[volumeID].(map[string]interface{}); ok { newAnnotation, err := r.ensureVolumeTags(ec2svc, aws.String(volumeID), subAnnotation, machine.Spec.AdditionalTags) if err != nil { r.Log.Error(err, "Failed to fetch the changed volume tags in EC2 instance") } - annotations[volumeID] = newAnnotation + machineAnnotations[volumeID] = newAnnotation } else { newAnnotation, err := r.ensureVolumeTags(ec2svc, aws.String(volumeID), make(map[string]interface{}), machine.Spec.AdditionalTags) if err != nil { r.Log.Error(err, "Failed to fetch the changed volume tags in EC2 instance") } - annotations[volumeID] = newAnnotation + machineAnnotations[volumeID] = newAnnotation } // We also need to update the annotation if anything changed. - err = r.updateMachineAnnotationJSON(machine, VolumeTagsLastAppliedAnnotation, annotations) + err = r.updateMachineAnnotationJSON(machine, VolumeTagsLastAppliedAnnotation, machineAnnotations) if err != nil { r.Log.Error(err, "Failed to fetch the changed volume tags in EC2 instance") } diff --git a/controllers/awsmachine_controller_test.go b/controllers/awsmachine_controller_test.go index f13d892539..854c9ed268 100644 --- a/controllers/awsmachine_controller_test.go +++ b/controllers/awsmachine_controller_test.go @@ -582,7 +582,7 @@ func mockedCreateInstanceCalls(m *mocks.MockEC2APIMockRecorder) { }, }, }, - }}, nil).MaxTimes(2) + }}, nil).MaxTimes(3) m.DescribeNetworkInterfaceAttribute(gomock.Eq(&ec2.DescribeNetworkInterfaceAttributeInput{ NetworkInterfaceId: aws.String("eni-1"), Attribute: aws.String("groupSet"), diff --git a/controlplane/eks/api/v1alpha3/conversion.go b/controlplane/eks/api/v1alpha3/conversion.go index eb1f04bd97..cd2ddcd02c 100644 --- a/controlplane/eks/api/v1alpha3/conversion.go +++ b/controlplane/eks/api/v1alpha3/conversion.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha3 import ( + "fmt" + apiconversion "k8s.io/apimachinery/pkg/conversion" infrav1alpha3 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha3" infrav1beta1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -44,6 +46,27 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { dst.Status.Bastion = restored.Status.Bastion dst.Spec.OIDCIdentityProviderConfig = restored.Spec.OIDCIdentityProviderConfig dst.Spec.KubeProxy = restored.Spec.KubeProxy + dst.Spec.VpcCni = restored.Spec.VpcCni + + dst.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID = restored.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID + dst.Spec.NetworkSpec.VPC.EnableIPv6 = restored.Spec.NetworkSpec.VPC.EnableIPv6 + dst.Spec.NetworkSpec.VPC.IPv6CidrBlock = restored.Spec.NetworkSpec.VPC.IPv6CidrBlock + dst.Spec.NetworkSpec.VPC.IPv6Pool = restored.Spec.NetworkSpec.VPC.IPv6Pool + + for i := range dst.Spec.NetworkSpec.Subnets { + var found bool + for k := range restored.Spec.NetworkSpec.Subnets { + if dst.Spec.NetworkSpec.Subnets[i].ID == restored.Spec.NetworkSpec.Subnets[k].ID { + dst.Spec.NetworkSpec.Subnets[i].IsIPv6 = restored.Spec.NetworkSpec.Subnets[i].IsIPv6 + dst.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock = restored.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock + found = true + break + } + } + if !found { + return fmt.Errorf("subnet with id %s not found amongts restored subnets", dst.Spec.NetworkSpec.Subnets[i].ID) + } + } return nil } @@ -134,3 +157,15 @@ func Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha3_AWSManagedControlPla func Convert_v1beta1_AWSManagedControlPlaneStatus_To_v1alpha3_AWSManagedControlPlaneStatus(in *v1beta1.AWSManagedControlPlaneStatus, out *AWSManagedControlPlaneStatus, scope apiconversion.Scope) error { return autoConvert_v1beta1_AWSManagedControlPlaneStatus_To_v1alpha3_AWSManagedControlPlaneStatus(in, out, scope) } + +func Convert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *infrav1beta1.OIDCProviderStatus, _ apiconversion.Scope) error { + out.ARN = in.ARN + out.TrustPolicy = in.TrustPolicy + return nil +} + +func Convert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(in *infrav1beta1.OIDCProviderStatus, out *OIDCProviderStatus, _ apiconversion.Scope) error { + out.ARN = in.ARN + out.TrustPolicy = in.TrustPolicy + return nil +} diff --git a/controlplane/eks/api/v1alpha3/zz_generated.conversion.go b/controlplane/eks/api/v1alpha3/zz_generated.conversion.go index c73a0173d7..49160eb966 100644 --- a/controlplane/eks/api/v1alpha3/zz_generated.conversion.go +++ b/controlplane/eks/api/v1alpha3/zz_generated.conversion.go @@ -150,16 +150,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*OIDCProviderStatus)(nil), (*v1beta1.OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(a.(*OIDCProviderStatus), b.(*v1beta1.OIDCProviderStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.OIDCProviderStatus)(nil), (*OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(a.(*v1beta1.OIDCProviderStatus), b.(*OIDCProviderStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*RoleMapping)(nil), (*v1beta1.RoleMapping)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_RoleMapping_To_v1beta1_RoleMapping(a.(*RoleMapping), b.(*v1beta1.RoleMapping), scope) }); err != nil { @@ -195,6 +185,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*OIDCProviderStatus)(nil), (*apiv1beta1.OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(a.(*OIDCProviderStatus), b.(*apiv1beta1.OIDCProviderStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AWSManagedControlPlaneSpec)(nil), (*AWSManagedControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha3_AWSManagedControlPlaneSpec(a.(*v1beta1.AWSManagedControlPlaneSpec), b.(*AWSManagedControlPlaneSpec), scope) }); err != nil { @@ -215,6 +210,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*apiv1beta1.OIDCProviderStatus)(nil), (*OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(a.(*apiv1beta1.OIDCProviderStatus), b.(*OIDCProviderStatus), scope) + }); err != nil { + return err + } return nil } @@ -340,6 +340,7 @@ func autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha3_AWSManagedContro } out.SecondaryCidrBlock = (*string)(unsafe.Pointer(in.SecondaryCidrBlock)) out.Region = in.Region + // WARNING: in.Partition requires manual conversion: does not exist in peer-type out.SSHKeyName = (*string)(unsafe.Pointer(in.SSHKeyName)) out.Version = (*string)(unsafe.Pointer(in.Version)) out.RoleName = (*string)(unsafe.Pointer(in.RoleName)) @@ -365,6 +366,7 @@ func autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha3_AWSManagedContro out.Addons = (*[]Addon)(unsafe.Pointer(in.Addons)) // WARNING: in.OIDCIdentityProviderConfig requires manual conversion: does not exist in peer-type out.DisableVPCCNI = in.DisableVPCCNI + // WARNING: in.VpcCni requires manual conversion: does not exist in peer-type // WARNING: in.KubeProxy requires manual conversion: does not exist in peer-type return nil } @@ -673,28 +675,6 @@ func Convert_v1beta1_KubernetesMapping_To_v1alpha3_KubernetesMapping(in *v1beta1 return autoConvert_v1beta1_KubernetesMapping_To_v1alpha3_KubernetesMapping(in, out, s) } -func autoConvert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *v1beta1.OIDCProviderStatus, s conversion.Scope) error { - out.ARN = in.ARN - out.TrustPolicy = in.TrustPolicy - return nil -} - -// Convert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus is an autogenerated conversion function. -func Convert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *v1beta1.OIDCProviderStatus, s conversion.Scope) error { - return autoConvert_v1alpha3_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in, out, s) -} - -func autoConvert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(in *v1beta1.OIDCProviderStatus, out *OIDCProviderStatus, s conversion.Scope) error { - out.ARN = in.ARN - out.TrustPolicy = in.TrustPolicy - return nil -} - -// Convert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus is an autogenerated conversion function. -func Convert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(in *v1beta1.OIDCProviderStatus, out *OIDCProviderStatus, s conversion.Scope) error { - return autoConvert_v1beta1_OIDCProviderStatus_To_v1alpha3_OIDCProviderStatus(in, out, s) -} - func autoConvert_v1alpha3_RoleMapping_To_v1beta1_RoleMapping(in *RoleMapping, out *v1beta1.RoleMapping, s conversion.Scope) error { out.RoleARN = in.RoleARN if err := Convert_v1alpha3_KubernetesMapping_To_v1beta1_KubernetesMapping(&in.KubernetesMapping, &out.KubernetesMapping, s); err != nil { diff --git a/controlplane/eks/api/v1alpha4/conversion.go b/controlplane/eks/api/v1alpha4/conversion.go index 22d0cc1d9e..9f4b6a15be 100644 --- a/controlplane/eks/api/v1alpha4/conversion.go +++ b/controlplane/eks/api/v1alpha4/conversion.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha4 import ( + "fmt" + apiconversion "k8s.io/apimachinery/pkg/conversion" infrav1alpha4 "sigs.k8s.io/cluster-api-provider-aws/api/v1alpha4" infrav1beta1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -41,6 +43,26 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error { } dst.Spec.KubeProxy = restored.Spec.KubeProxy + dst.Spec.VpcCni = restored.Spec.VpcCni + dst.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID = restored.Spec.NetworkSpec.VPC.EgressOnlyInternetGatewayID + dst.Spec.NetworkSpec.VPC.EnableIPv6 = restored.Spec.NetworkSpec.VPC.EnableIPv6 + dst.Spec.NetworkSpec.VPC.IPv6CidrBlock = restored.Spec.NetworkSpec.VPC.IPv6CidrBlock + dst.Spec.NetworkSpec.VPC.IPv6Pool = restored.Spec.NetworkSpec.VPC.IPv6Pool + + for i := range dst.Spec.NetworkSpec.Subnets { + var found bool + for k := range restored.Spec.NetworkSpec.Subnets { + if dst.Spec.NetworkSpec.Subnets[i].ID == restored.Spec.NetworkSpec.Subnets[k].ID { + dst.Spec.NetworkSpec.Subnets[i].IsIPv6 = restored.Spec.NetworkSpec.Subnets[i].IsIPv6 + dst.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock = restored.Spec.NetworkSpec.Subnets[i].IPv6CidrBlock + found = true + break + } + } + if !found { + return fmt.Errorf("subnet with id %s not found amongts restored subnets", dst.Spec.NetworkSpec.Subnets[i].ID) + } + } return nil } @@ -74,10 +96,6 @@ func (r *AWSManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) error { return Convert_v1beta1_AWSManagedControlPlaneList_To_v1alpha4_AWSManagedControlPlaneList(src, r, nil) } -func Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in *v1beta1.AWSManagedControlPlaneSpec, out *AWSManagedControlPlaneSpec, scope apiconversion.Scope) error { - return autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in, out, scope) -} - // Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus is a conversion function. func Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus(in *infrav1alpha4.NetworkStatus, out *infrav1beta1.NetworkStatus, s apiconversion.Scope) error { return infrav1alpha4.Convert_v1alpha4_NetworkStatus_To_v1beta1_NetworkStatus(in, out, s) @@ -127,3 +145,19 @@ func Convert_v1beta1_Instance_To_v1alpha4_Instance(in *infrav1beta1.Instance, ou func Convert_v1alpha4_Instance_To_v1beta1_Instance(in *infrav1alpha4.Instance, out *infrav1beta1.Instance, s apiconversion.Scope) error { return infrav1alpha4.Convert_v1alpha4_Instance_To_v1beta1_Instance(in, out, s) } + +func Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in *v1beta1.AWSManagedControlPlaneSpec, out *AWSManagedControlPlaneSpec, scope apiconversion.Scope) error { + return autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(in, out, scope) +} + +func Convert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *infrav1beta1.OIDCProviderStatus, _ apiconversion.Scope) error { + out.ARN = in.ARN + out.TrustPolicy = in.TrustPolicy + return nil +} + +func Convert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(in *infrav1beta1.OIDCProviderStatus, out *OIDCProviderStatus, _ apiconversion.Scope) error { + out.ARN = in.ARN + out.TrustPolicy = in.TrustPolicy + return nil +} diff --git a/controlplane/eks/api/v1alpha4/conversion_test.go b/controlplane/eks/api/v1alpha4/conversion_test.go index f99f393f5b..256f123a20 100644 --- a/controlplane/eks/api/v1alpha4/conversion_test.go +++ b/controlplane/eks/api/v1alpha4/conversion_test.go @@ -21,8 +21,8 @@ import ( . "github.com/onsi/gomega" - runtime "k8s.io/apimachinery/pkg/runtime" - v1beta1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" ) diff --git a/controlplane/eks/api/v1alpha4/zz_generated.conversion.go b/controlplane/eks/api/v1alpha4/zz_generated.conversion.go index bf4dbff766..effafafd5a 100644 --- a/controlplane/eks/api/v1alpha4/zz_generated.conversion.go +++ b/controlplane/eks/api/v1alpha4/zz_generated.conversion.go @@ -175,16 +175,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*OIDCProviderStatus)(nil), (*v1beta1.OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(a.(*OIDCProviderStatus), b.(*v1beta1.OIDCProviderStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta1.OIDCProviderStatus)(nil), (*OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(a.(*v1beta1.OIDCProviderStatus), b.(*OIDCProviderStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*RoleMapping)(nil), (*v1beta1.RoleMapping)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_RoleMapping_To_v1beta1_RoleMapping(a.(*RoleMapping), b.(*v1beta1.RoleMapping), scope) }); err != nil { @@ -225,6 +215,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*OIDCProviderStatus)(nil), (*apiv1beta1.OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(a.(*OIDCProviderStatus), b.(*apiv1beta1.OIDCProviderStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AWSManagedControlPlaneSpec)(nil), (*AWSManagedControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedControlPlaneSpec(a.(*v1beta1.AWSManagedControlPlaneSpec), b.(*AWSManagedControlPlaneSpec), scope) }); err != nil { @@ -250,6 +245,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*apiv1beta1.OIDCProviderStatus)(nil), (*OIDCProviderStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(a.(*apiv1beta1.OIDCProviderStatus), b.(*OIDCProviderStatus), scope) + }); err != nil { + return err + } return nil } @@ -376,6 +376,7 @@ func autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedContro } out.SecondaryCidrBlock = (*string)(unsafe.Pointer(in.SecondaryCidrBlock)) out.Region = in.Region + // WARNING: in.Partition requires manual conversion: does not exist in peer-type out.SSHKeyName = (*string)(unsafe.Pointer(in.SSHKeyName)) out.Version = (*string)(unsafe.Pointer(in.Version)) out.RoleName = (*string)(unsafe.Pointer(in.RoleName)) @@ -401,6 +402,7 @@ func autoConvert_v1beta1_AWSManagedControlPlaneSpec_To_v1alpha4_AWSManagedContro out.Addons = (*[]Addon)(unsafe.Pointer(in.Addons)) out.OIDCIdentityProviderConfig = (*OIDCIdentityProviderConfig)(unsafe.Pointer(in.OIDCIdentityProviderConfig)) out.DisableVPCCNI = in.DisableVPCCNI + // WARNING: in.VpcCni requires manual conversion: does not exist in peer-type // WARNING: in.KubeProxy requires manual conversion: does not exist in peer-type return nil } @@ -777,28 +779,6 @@ func Convert_v1beta1_OIDCIdentityProviderConfig_To_v1alpha4_OIDCIdentityProvider return autoConvert_v1beta1_OIDCIdentityProviderConfig_To_v1alpha4_OIDCIdentityProviderConfig(in, out, s) } -func autoConvert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *v1beta1.OIDCProviderStatus, s conversion.Scope) error { - out.ARN = in.ARN - out.TrustPolicy = in.TrustPolicy - return nil -} - -// Convert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus is an autogenerated conversion function. -func Convert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in *OIDCProviderStatus, out *v1beta1.OIDCProviderStatus, s conversion.Scope) error { - return autoConvert_v1alpha4_OIDCProviderStatus_To_v1beta1_OIDCProviderStatus(in, out, s) -} - -func autoConvert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(in *v1beta1.OIDCProviderStatus, out *OIDCProviderStatus, s conversion.Scope) error { - out.ARN = in.ARN - out.TrustPolicy = in.TrustPolicy - return nil -} - -// Convert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus is an autogenerated conversion function. -func Convert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(in *v1beta1.OIDCProviderStatus, out *OIDCProviderStatus, s conversion.Scope) error { - return autoConvert_v1beta1_OIDCProviderStatus_To_v1alpha4_OIDCProviderStatus(in, out, s) -} - func autoConvert_v1alpha4_RoleMapping_To_v1beta1_RoleMapping(in *RoleMapping, out *v1beta1.RoleMapping, s conversion.Scope) error { out.RoleARN = in.RoleARN if err := Convert_v1alpha4_KubernetesMapping_To_v1beta1_KubernetesMapping(&in.KubernetesMapping, &out.KubernetesMapping, s); err != nil { diff --git a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_types.go b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_types.go index db7df38549..fca8acba78 100644 --- a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_types.go +++ b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -51,6 +52,10 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // The AWS Region the cluster lives in. Region string `json:"region,omitempty"` + // Partition is the AWS security partition being used. Defaults to "aws" + // +optional + Partition string `json:"partition,omitempty"` + // SSHKeyName is the name of the ssh key to attach to the bastion host. Valid values are empty string (do not use SSH keys), a valid SSH key name, or omitted (use the default SSH key name) // +optional SSHKeyName *string `json:"sshKeyName,omitempty"` @@ -167,6 +172,10 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned // +kubebuilder:default=false DisableVPCCNI bool `json:"disableVPCCNI,omitempty"` + // VpcCni is used to set configuration options for the VPC CNI plugin + // +optional + VpcCni VpcCni `json:"VpcCni,omitempty"` + // KubeProxy defines managed attributes of the kube-proxy daemonset KubeProxy KubeProxy `json:"kubeProxy,omitempty"` } @@ -182,6 +191,13 @@ type KubeProxy struct { Disable bool `json:"disable,omitempty"` } +// VpcCni specifies configuration related to the VPC CNI. +type VpcCni struct { + // Env defines a list of environment variables to apply to the `aws-node` DaemonSet + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` +} + // EndpointAccess specifies how control plane endpoints are accessible. type EndpointAccess struct { // Public controls whether control plane endpoints are publicly accessible @@ -203,14 +219,6 @@ type EncryptionConfig struct { Resources []*string `json:"resources,omitempty"` } -// OIDCProviderStatus holds the status of the AWS OIDC identity provider. -type OIDCProviderStatus struct { - // ARN holds the ARN of the provider - ARN string `json:"arn,omitempty"` - // TrustPolicy contains the boilerplate IAM trust policy to use for IRSA - TrustPolicy string `json:"trustPolicy,omitempty"` -} - type IdentityProviderStatus struct { // ARN holds the ARN of associated identity provider ARN string `json:"arn,omitempty"` @@ -232,7 +240,7 @@ type AWSManagedControlPlaneStatus struct { Bastion *infrav1.Instance `json:"bastion,omitempty"` // OIDCProvider holds the status of the identity provider for this cluster // +optional - OIDCProvider OIDCProviderStatus `json:"oidcProvider,omitempty"` + OIDCProvider infrav1.OIDCProviderStatus `json:"oidcProvider,omitempty"` // ExternalManagedControlPlane indicates to cluster-api that the control plane // is managed by an external service such as AKS, EKS, GKE, etc. // +kubebuilder:default=true diff --git a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook.go b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook.go index 98718f3b01..ee77351b02 100644 --- a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook.go +++ b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook.go @@ -35,8 +35,10 @@ import ( ) const ( - minAddonVersion = "v1.18.0" - maxClusterNameLength = 100 + minAddonVersion = "v1.18.0" + minKubeVersionForIPv6 = "v1.21.0" + minVpcCniVersionForIPv6 = "1.10.2" + maxClusterNameLength = 100 ) // log is for logging in this package. @@ -89,6 +91,7 @@ func (r *AWSManagedControlPlane) ValidateCreate() error { allErrs = append(allErrs, field.Required(field.NewPath("spec.eksClusterName"), "eksClusterName is required")) } + // TODO: Add ipv6 validation things in these validations. allErrs = append(allErrs, r.validateEKSVersion(nil)...) allErrs = append(allErrs, r.Spec.Bastion.Validate()...) allErrs = append(allErrs, r.validateIAMAuthConfig()...) @@ -97,6 +100,7 @@ func (r *AWSManagedControlPlane) ValidateCreate() error { allErrs = append(allErrs, r.validateDisableVPCCNI()...) allErrs = append(allErrs, r.validateKubeProxy()...) allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...) + allErrs = append(allErrs, r.validateNetwork()...) if len(allErrs) == 0 { return nil @@ -163,6 +167,11 @@ func (r *AWSManagedControlPlane) ValidateUpdate(old runtime.Object) error { ) } + if oldAWSManagedControlplane.Spec.NetworkSpec.VPC.EnableIPv6 != r.Spec.NetworkSpec.VPC.EnableIPv6 { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec", "networkSpec", "vpc", "enableIPv6"), r.Spec.NetworkSpec.VPC.EnableIPv6, "changing IP family is not allowed after it has been set")) + } + if len(allErrs) == 0 { return nil } @@ -220,13 +229,19 @@ func (r *AWSManagedControlPlane) validateEKSVersion(old *AWSManagedControlPlane) } } + if r.Spec.NetworkSpec.VPC.EnableIPv6 { + minIPv6, _ := version.ParseSemantic(minKubeVersionForIPv6) + if v.LessThan(minIPv6) { + allErrs = append(allErrs, field.Invalid(path, *r.Spec.Version, fmt.Sprintf("IPv6 requires Kubernetes %s or greater", minKubeVersionForIPv6))) + } + } return allErrs } func (r *AWSManagedControlPlane) validateEKSAddons() field.ErrorList { var allErrs field.ErrorList - if r.Spec.Addons == nil || len(*r.Spec.Addons) == 0 { + if !r.Spec.NetworkSpec.VPC.EnableIPv6 && (r.Spec.Addons == nil || len(*r.Spec.Addons) == 0) { return allErrs } @@ -245,6 +260,31 @@ func (r *AWSManagedControlPlane) validateEKSAddons() field.ErrorList { allErrs = append(allErrs, field.Invalid(addonsPath, *r.Spec.Version, message)) } + // validations for IPv6: + // - addons have to be defined in case IPv6 is enabled + // - minimum version requirement for VPC-CNI using IPv6 ipFamily is 1.10.2 + if r.Spec.NetworkSpec.VPC.EnableIPv6 { + if r.Spec.Addons == nil || len(*r.Spec.Addons) == 0 { + allErrs = append(allErrs, field.Invalid(addonsPath, "", "addons are required to be set explicitly if IPv6 is enabled")) + return allErrs + } + + for _, addon := range *r.Spec.Addons { + if addon.Name == vpcCniAddon { + v, err := version.ParseGeneric(addon.Version) + if err != nil { + allErrs = append(allErrs, field.Invalid(addonsPath, addon.Version, err.Error())) + break + } + minCniVersion, _ := version.ParseSemantic(minVpcCniVersionForIPv6) + if v.LessThan(minCniVersion) { + allErrs = append(allErrs, field.Invalid(addonsPath, addon.Version, fmt.Sprintf("vpc-cni version must be above or equal to %s for IPv6", minVpcCniVersionForIPv6))) + break + } + } + } + } + return allErrs } @@ -353,6 +393,17 @@ func (r *AWSManagedControlPlane) validateDisableVPCCNI() field.ErrorList { return allErrs } +func (r *AWSManagedControlPlane) validateNetwork() field.ErrorList { + var allErrs field.ErrorList + + if r.Spec.NetworkSpec.VPC.EnableIPv6 && r.Spec.NetworkSpec.VPC.IPv6CidrBlock != "" && r.Spec.NetworkSpec.VPC.IPv6Pool == "" { + poolField := field.NewPath("spec", "networkSpec", "vpc", "ipv6Pool") + allErrs = append(allErrs, field.Invalid(poolField, r.Spec.NetworkSpec.VPC.IPv6Pool, "ipv6Pool cannot be empty if ipv6CidrBlock is set for BYOIP")) + } + + return allErrs +} + // Default will set default values for the AWSManagedControlPlane. func (r *AWSManagedControlPlane) Default() { mcpLog.Info("AWSManagedControlPlane setting defaults", "name", r.Name) diff --git a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook_test.go b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook_test.go index fe9ea9faf6..100e05805c 100644 --- a/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook_test.go +++ b/controlplane/eks/api/v1beta1/awsmanagedcontrolplane_webhook_test.go @@ -18,6 +18,7 @@ package v1beta1 import ( "context" + "fmt" "strings" "testing" @@ -359,6 +360,107 @@ func TestWebhookCreate(t *testing.T) { } } +func TestWebhookCreate_IPv6Details(t *testing.T) { + tests := []struct { + name string + addons []Addon + kubeVersion string + networkSpec infrav1.NetworkSpec + err string + }{ + { + name: "ipv6 with lower cluster version", + kubeVersion: "v1.18", + err: fmt.Sprintf("IPv6 requires Kubernetes %s or greater", minKubeVersionForIPv6), + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + }, + { + name: "ipv6 no addons", + kubeVersion: "v1.22", + err: "addons are required to be set explicitly if IPv6 is enabled", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + }, + { + name: "ipv6 with addons but cni version is lower than supported version", + kubeVersion: "v1.22", + addons: []Addon{ + { + Name: vpcCniAddon, + Version: "1.9.3", + }, + }, + err: fmt.Sprintf("vpc-cni version must be above or equal to %s for IPv6", minVpcCniVersionForIPv6), + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + }, + { + name: "ipv6 with addons and correct cni and cluster version", + kubeVersion: "v1.22", + addons: []Addon{ + { + Name: vpcCniAddon, + Version: "1.11.0", + }, + }, + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + }, + { + name: "ipv6 with BYOIP but pool is left empty", + kubeVersion: "v1.18", + networkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + IPv6CidrBlock: "not-empty", + // IPv6Pool is empty + }, + }, + err: "ipv6Pool cannot be empty if ipv6CidrBlock is set for BYOIP", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.TODO() + g := NewWithT(t) + + mcp := &AWSManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "mcp-", + Namespace: "default", + }, + Spec: AWSManagedControlPlaneSpec{ + EKSClusterName: "test-cluster", + Addons: &tc.addons, + NetworkSpec: tc.networkSpec, + Version: &tc.kubeVersion, + }, + } + err := testEnv.Create(ctx, mcp) + + if tc.err != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.err))) + } else { + g.Expect(err).To(BeNil()) + } + }) + } +} + func TestWebhookUpdate(t *testing.T) { tests := []struct { name string @@ -514,6 +616,53 @@ func TestWebhookUpdate(t *testing.T) { }, expectError: true, }, + { + name: "changing ipv6 enabled is not allowed after it has been set - false, true", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + NetworkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: false, + }, + }, + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + NetworkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + }, + expectError: true, + }, + { + name: "changing ipv6 enabled is not allowed after it has been set - true, false", + oldClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + NetworkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + }, + }, + Addons: &[]Addon{ + { + Name: vpcCniAddon, + Version: "1.11.0", + }, + }, + Version: pointer.String("v1.22.0"), + }, + newClusterSpec: AWSManagedControlPlaneSpec{ + EKSClusterName: "default_cluster1", + NetworkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: false, + }, + }, + }, + expectError: true, + }, } for _, tc := range tests { diff --git a/controlplane/eks/api/v1beta1/zz_generated.deepcopy.go b/controlplane/eks/api/v1beta1/zz_generated.deepcopy.go index edda9bada4..48c0c108be 100644 --- a/controlplane/eks/api/v1beta1/zz_generated.deepcopy.go +++ b/controlplane/eks/api/v1beta1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ limitations under the License. package v1beta1 import ( + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" apiv1beta1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" cluster_apiapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -170,6 +171,7 @@ func (in *AWSManagedControlPlaneSpec) DeepCopyInto(out *AWSManagedControlPlaneSp *out = new(OIDCIdentityProviderConfig) (*in).DeepCopyInto(*out) } + in.VpcCni.DeepCopyInto(&out.VpcCni) out.KubeProxy = in.KubeProxy } @@ -537,48 +539,55 @@ func (in *OIDCIdentityProviderConfig) DeepCopy() *OIDCIdentityProviderConfig { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OIDCProviderStatus) DeepCopyInto(out *OIDCProviderStatus) { +func (in *RoleMapping) DeepCopyInto(out *RoleMapping) { *out = *in + in.KubernetesMapping.DeepCopyInto(&out.KubernetesMapping) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OIDCProviderStatus. -func (in *OIDCProviderStatus) DeepCopy() *OIDCProviderStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleMapping. +func (in *RoleMapping) DeepCopy() *RoleMapping { if in == nil { return nil } - out := new(OIDCProviderStatus) + out := new(RoleMapping) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RoleMapping) DeepCopyInto(out *RoleMapping) { +func (in *UserMapping) DeepCopyInto(out *UserMapping) { *out = *in in.KubernetesMapping.DeepCopyInto(&out.KubernetesMapping) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleMapping. -func (in *RoleMapping) DeepCopy() *RoleMapping { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserMapping. +func (in *UserMapping) DeepCopy() *UserMapping { if in == nil { return nil } - out := new(RoleMapping) + out := new(UserMapping) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *UserMapping) DeepCopyInto(out *UserMapping) { +func (in *VpcCni) DeepCopyInto(out *VpcCni) { *out = *in - in.KubernetesMapping.DeepCopyInto(&out.KubernetesMapping) + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserMapping. -func (in *UserMapping) DeepCopy() *UserMapping { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcCni. +func (in *VpcCni) DeepCopy() *VpcCni { if in == nil { return nil } - out := new(UserMapping) + out := new(VpcCni) in.DeepCopyInto(out) return out } diff --git a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go index af06f47c26..e27c99eee3 100644 --- a/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go +++ b/controlplane/eks/controllers/awsmanagedcontrolplane_controller.go @@ -103,7 +103,7 @@ func (r *AWSManagedControlPlaneReconciler) SetupWithManager(ctx context.Context, if err = c.Watch( &source.Kind{Type: &clusterv1.Cluster{}}, - handler.EnqueueRequestsFromMapFunc(util.ClusterToInfrastructureMapFunc(awsManagedControlPlane.GroupVersionKind())), + handler.EnqueueRequestsFromMapFunc(util.ClusterToInfrastructureMapFunc(ctx, awsManagedControlPlane.GroupVersionKind(), mgr.GetClient(), &ekscontrolplanev1.AWSManagedControlPlane{})), predicates.ClusterUnpausedAndInfrastructureReady(log), ); err != nil { return fmt.Errorf("failed adding a watch for ready clusters: %w", err) @@ -186,6 +186,9 @@ func (r *AWSManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ct if managedScope.Bastion().Enabled { applicableConditions = append(applicableConditions, infrav1.BastionHostReadyCondition) } + if managedScope.VPC().EnableIPv6 { + applicableConditions = append(applicableConditions, infrav1.EgressOnlyInternetGatewayReadyCondition) + } } conditions.SetSummary(managedScope.ControlPlane, conditions.WithConditions(applicableConditions...), conditions.WithStepCounter()) diff --git a/docs/book/src/topics/eks/ipv6-enabled-cluster.md b/docs/book/src/topics/eks/ipv6-enabled-cluster.md new file mode 100644 index 0000000000..25fdf150e8 --- /dev/null +++ b/docs/book/src/topics/eks/ipv6-enabled-cluster.md @@ -0,0 +1,77 @@ +# IPv6 Enabled Cluster + +CAPA supports IPv6 enabled clusters. Dual stack clusters are not yet supported, but +dual VPC, meaning both ipv6 and ipv4 are defined, is support and in fact, it's the +only mode of operation at the writing of this doc. + +Upcoming feature will be IPv6 _only_. + +## How to set up + +Two modes of operations are supported. Request AWS to generate and assign an address +or BYOIP which is Bring Your Own IP. There must already be a provisioned pool and a +set of IPv6 CIDR for that. + +### Automatically Generated IP + +To request AWS to assign a set of IPv6 addresses from an AWS defined address pool, +use the following setting: + +```yaml +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + network: + vpc: + enableIPv6: true +``` + +### BYOIP ( Bring Your Own IP ) + +To define your own IPv6 address pool and CIDR set the following values: + +```yaml +spec: + network: + vpc: + ipv6Pool: pool-id + ipv6CidrBlock: "2009:1234:ff00::/56" + enableIPv6: true +``` + +## Requirements + +The use of a Nitro enabled instance is required. To see a list of nitro instances in your region +run the following command: + +```bash +aws ec2 describe-instance-types --filters Name=hypervisor,Values=nitro --region us-west-2 | grep "InstanceType" +``` + +This will list all available Nitro hypervisor based instances in your region. + +All addons **must** be enabled. A working cluster configuration looks like this: + +```yaml +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + network: + vpc: + enableIPv6: true + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + addons: + - name: "vpc-cni" + version: "v1.11.0-eksbuild.1" + conflictResolution: "overwrite" # this is important, otherwise environment property update will not work + - name: "coredns" + version: "v1.8.7-eksbuild.1" + - name: "kube-proxy" + version: "v1.22.6-eksbuild.1" +``` diff --git a/docs/book/src/topics/eks/pod-networking.md b/docs/book/src/topics/eks/pod-networking.md index d3247fd0d9..23d87d96dc 100644 --- a/docs/book/src/topics/eks/pod-networking.md +++ b/docs/book/src/topics/eks/pod-networking.md @@ -7,6 +7,48 @@ When creating a EKS cluster the Amazon VPC CNI will be used by default for Pod N ## Using the VPC CNI Addon You can use an explicit version of the Amazon VPC CNI by using the **vpc-cni** EKS addon. See the [addons](./addons.md) documentation for further details of how to use addons. +## Using Custom VPC CNI Configuration +If your use case demands [custom networking](https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html) VPC CNI configuration you might already be familiar with the [helm chart](https://github.com/aws/amazon-vpc-cni-k8s) which helps with the process. This gives you access to ENI Configs and you can set Environment Variables on the `aws-node` DaemonSet where the VPC CNI runs. CAPA is able to tune the same DaemonSet through Kubernetes. + +The following example shows how to turn on custom network config and set a [label definition](https://github.com/aws/amazon-vpc-cni-k8s#eni_config_label_def). + +```yaml +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "capi-managed-test-control-plane" +spec: + vpcCni: + env: + - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG + value: "true" + - name: ENABLE_PREFIX_DELEGATION + value: "true" +``` + +### Increase node pod limit +You can increase the pod limit per-node as [per the upstream AWS documentation](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/). You'll need to enable the `vpc-cni` plugin addon on your EKS cluster as well as enable prefix assignment mode through the `ENABLE_PREFIX_DELEGATION` environment variable. + +```yaml +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "capi-managed-test-control-plane" +spec: + vpcCni: + env: + - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG + value: "true" + - name: ENABLE_PREFIX_DELEGATION + value: "true" + addons: + - name: vpc-cni + version: + conflictResolution: overwrite + associateOIDCProvider: true + disableVPCCNI: false +``` + ## Using an alternative CNI There may be scenarios where you do not want to use the Amazon VPC CNI. EKS supports a number of alternative CNIs such as Calico, Cilium, and Weave Net (see [docs](https://docs.aws.amazon.com/eks/latest/userguide/alternate-cni-plugins.html) for full list). diff --git a/exp/api/v1beta1/awsmachinepool_types.go b/exp/api/v1beta1/awsmachinepool_types.go index 55764b905b..46e98dd91c 100644 --- a/exp/api/v1beta1/awsmachinepool_types.go +++ b/exp/api/v1beta1/awsmachinepool_types.go @@ -126,6 +126,10 @@ type AWSMachinePoolStatus struct { // The ID of the launch template LaunchTemplateID string `json:"launchTemplateID,omitempty"` + // The version of the launch template + // +optional + LaunchTemplateVersion *string `json:"launchTemplateVersion,omitempty"` + // FailureReason will be set in the event that there is a terminal problem // reconciling the Machine and will contain a succinct value suitable // for machine interpretation. diff --git a/exp/api/v1beta1/awsmanagedmachinepool_types.go b/exp/api/v1beta1/awsmanagedmachinepool_types.go index 93f8ab73ca..e6e95d9a14 100644 --- a/exp/api/v1beta1/awsmanagedmachinepool_types.go +++ b/exp/api/v1beta1/awsmanagedmachinepool_types.go @@ -144,6 +144,12 @@ type AWSManagedMachinePoolSpec struct { // to the nodegroup. // +optional UpdateConfig *UpdateConfig `json:"updateConfig,omitempty"` + + // AWSLaunchTemplate specifies the launch template to use to create the managed node group. + // If AWSLaunchTemplate is specified, certain node group configuraions outside of launch template + // are prohibited (https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). + // +optional + AWSLaunchTemplate *AWSLaunchTemplate `json:"awsLaunchTemplate,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. @@ -176,6 +182,14 @@ type AWSManagedMachinePoolStatus struct { // +optional Replicas int32 `json:"replicas"` + // The ID of the launch template + // +optional + LaunchTemplateID *string `json:"launchTemplateID,omitempty"` + + // The version of the launch template + // +optional + LaunchTemplateVersion *string `json:"launchTemplateVersion,omitempty"` + // FailureReason will be set in the event that there is a terminal problem // reconciling the MachinePool and will contain a succinct value suitable // for machine interpretation. diff --git a/exp/api/v1beta1/awsmanagedmachinepool_webhook.go b/exp/api/v1beta1/awsmanagedmachinepool_webhook.go index edb5b67e94..8263519ac5 100644 --- a/exp/api/v1beta1/awsmanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/awsmanagedmachinepool_webhook.go @@ -117,6 +117,26 @@ func (r *AWSManagedMachinePool) validateRemoteAccess() field.ErrorList { return allErrs } +func (r *AWSManagedMachinePool) validateLaunchTemplate() field.ErrorList { + var allErrs field.ErrorList + if r.Spec.AWSLaunchTemplate == nil { + return allErrs + } + + if r.Spec.InstanceType != nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "InstanceType"), r.Spec.InstanceType, "InstanceType cannot be specified when LaunchTemplate is specified")) + } + if r.Spec.DiskSize != nil { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "DiskSize"), r.Spec.DiskSize, "DiskSize cannot be specified when LaunchTemplate is specified")) + } + + if r.Spec.AWSLaunchTemplate.IamInstanceProfile != "" { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "AWSLaunchTemplate", "IamInstanceProfile"), r.Spec.AWSLaunchTemplate.IamInstanceProfile, "IAM instance profile in launch template is prohibited in EKS managed node group")) + } + + return allErrs +} + // ValidateCreate will do any extra validation when creating a AWSManagedMachinePool. func (r *AWSManagedMachinePool) ValidateCreate() error { mmpLog.Info("AWSManagedMachinePool validate create", "name", r.Name) @@ -135,6 +155,9 @@ func (r *AWSManagedMachinePool) ValidateCreate() error { if errs := r.validateNodegroupUpdateConfig(); len(errs) > 0 { allErrs = append(allErrs, errs...) } + if errs := r.validateLaunchTemplate(); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...) @@ -169,6 +192,9 @@ func (r *AWSManagedMachinePool) ValidateUpdate(old runtime.Object) error { if errs := r.validateNodegroupUpdateConfig(); len(errs) > 0 { allErrs = append(allErrs, errs...) } + if errs := r.validateLaunchTemplate(); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } if len(allErrs) == 0 { return nil @@ -217,6 +243,16 @@ func (r *AWSManagedMachinePool) validateImmutable(old *AWSManagedMachinePool) fi appendErrorIfMutated(old.Spec.AMIType, r.Spec.AMIType, "amiType") appendErrorIfMutated(old.Spec.RemoteAccess, r.Spec.RemoteAccess, "remoteAccess") appendErrorIfSetAndMutated(old.Spec.CapacityType, r.Spec.CapacityType, "capacityType") + if (old.Spec.AWSLaunchTemplate != nil && r.Spec.AWSLaunchTemplate == nil) || + (old.Spec.AWSLaunchTemplate == nil && r.Spec.AWSLaunchTemplate != nil) { + allErrs = append( + allErrs, + field.Invalid(field.NewPath("spec", "AWSLaunchTemplate"), old.Spec.AWSLaunchTemplate, "field is immutable"), + ) + } + if old.Spec.AWSLaunchTemplate != nil && r.Spec.AWSLaunchTemplate != nil { + appendErrorIfMutated(old.Spec.AWSLaunchTemplate.Name, r.Spec.AWSLaunchTemplate.Name, "awsLaunchTemplate.name") + } return allErrs } diff --git a/exp/api/v1beta1/conditions_consts.go b/exp/api/v1beta1/conditions_consts.go index dcf44a1d7e..534ebb2bf9 100644 --- a/exp/api/v1beta1/conditions_consts.go +++ b/exp/api/v1beta1/conditions_consts.go @@ -34,6 +34,18 @@ const ( LaunchTemplateNotFoundReason = "LaunchTemplateNotFound" // LaunchTemplateCreateFailedReason used for failures during Launch Template creation. LaunchTemplateCreateFailedReason = "LaunchTemplateCreateFailed" + // LaunchTemplateReconcileFailedReason used for failures during Launch Template reconciliation. + LaunchTemplateReconcileFailedReason = "LaunchTemplateReconcileFailed" + + // PreLaunchTemplateUpdateCheckCondition reports if all prerequisite are met for launch template update. + PreLaunchTemplateUpdateCheckCondition clusterv1.ConditionType = "PreLaunchTemplateUpdateCheckSuccess" + // PostLaunchTemplateUpdateOperationCondition reports on successfully completes post launch template update operation. + PostLaunchTemplateUpdateOperationCondition clusterv1.ConditionType = "PostLaunchTemplateUpdateOperationSuccess" + + // PreLaunchTemplateUpdateCheckFailedReason used to report when not all prerequisite are met for launch template update. + PreLaunchTemplateUpdateCheckFailedReason = "PreLaunchTemplateUpdateCheckFailed" + // PostLaunchTemplateUpdateOperationFailedReason used to report when post launch template update operation failed. + PostLaunchTemplateUpdateOperationFailedReason = "PostLaunchTemplateUpdateOperationFailed" // InstanceRefreshStartedCondition reports on successfully starting instance refresh. InstanceRefreshStartedCondition clusterv1.ConditionType = "InstanceRefreshStarted" diff --git a/exp/controllers/awsmachinepool_controller.go b/exp/controllers/awsmachinepool_controller.go index d7637ec89b..29d5e44428 100644 --- a/exp/controllers/awsmachinepool_controller.go +++ b/exp/controllers/awsmachinepool_controller.go @@ -45,7 +45,6 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services" asg "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/autoscaling" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ec2" - "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/userdata" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -222,7 +221,29 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP return ctrl.Result{}, nil } - if err := r.reconcileLaunchTemplate(machinePoolScope, ec2Scope); err != nil { + ec2Svc := r.getEC2Service(ec2Scope) + asgsvc := r.getASGService(clusterScope) + + canUpdateLaunchTemplate := func() (bool, error) { + // If there is a change: before changing the template, check if there exist an ongoing instance refresh, + // because only 1 instance refresh can be "InProgress". If template is updated when refresh cannot be started, + // that change will not trigger a refresh. Do not start an instance refresh if only userdata changed. + return asgsvc.CanStartASGInstanceRefresh(machinePoolScope) + } + runPostLaunchTemplateUpdateOperation := func() error { + // After creating a new version of launch template, instance refresh is required + // to trigger a rolling replacement of all previously launched instances. + // If ONLY the userdata changed, previously launched instances continue to use the old launch + // template. + // + // FIXME(dlipovetsky,sedefsavas): If the controller terminates, or the StartASGInstanceRefresh returns an error, + // this conditional will not evaluate to true the next reconcile. If any machines use an older + // Launch Template version, and the difference between the older and current versions is _more_ + // than userdata, we should start an Instance Refresh. + machinePoolScope.Info("starting instance refresh", "number of instances", machinePoolScope.MachinePool.Spec.Replicas) + return asgsvc.StartASGInstanceRefresh(machinePoolScope) + } + if err := ec2Svc.ReconcileLaunchTemplate(machinePoolScope, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil { r.Recorder.Eventf(machinePoolScope.AWSMachinePool, corev1.EventTypeWarning, "FailedLaunchTemplateReconcile", "Failed to reconcile launch template: %v", err) machinePoolScope.Error(err, "failed to reconcile launch template") return ctrl.Result{}, err @@ -231,9 +252,6 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP // set the LaunchTemplateReady condition conditions.MarkTrue(machinePoolScope.AWSMachinePool, expinfrav1.LaunchTemplateReadyCondition) - // Initialize ASG client - asgsvc := r.getASGService(clusterScope) - // Find existing ASG asg, err := r.findASG(machinePoolScope, asgsvc) if err != nil { @@ -255,7 +273,19 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP return ctrl.Result{}, err } - err = r.reconcileTags(machinePoolScope, clusterScope, ec2Scope) + launchTemplateID := machinePoolScope.GetLaunchTemplateIDStatus() + asgName := machinePoolScope.Name() + resourceServiceToUpdate := []scope.ResourceServiceToUpdate{ + { + ResourceID: &launchTemplateID, + ResourceService: ec2Svc, + }, + { + ResourceID: &asgName, + ResourceService: asgsvc, + }, + } + err = ec2Svc.ReconcileTags(machinePoolScope, resourceServiceToUpdate) if err != nil { return ctrl.Result{}, errors.Wrap(err, "error updating tags") } @@ -316,7 +346,7 @@ func (r *AWSMachinePoolReconciler) reconcileDelete(machinePoolScope *scope.Machi } launchTemplateID := machinePoolScope.AWSMachinePool.Status.LaunchTemplateID - launchTemplate, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.Name()) + launchTemplate, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName()) if err != nil { return ctrl.Result{}, err } @@ -380,134 +410,6 @@ func (r *AWSMachinePoolReconciler) findASG(machinePoolScope *scope.MachinePoolSc return asg, nil } -func (r *AWSMachinePoolReconciler) reconcileLaunchTemplate(machinePoolScope *scope.MachinePoolScope, ec2Scope scope.EC2Scope) error { - bootstrapData, err := machinePoolScope.GetRawBootstrapData() - if err != nil { - r.Recorder.Eventf(machinePoolScope.AWSMachinePool, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error()) - } - bootstrapDataHash := userdata.ComputeHash(bootstrapData) - - ec2svc := r.getEC2Service(ec2Scope) - - machinePoolScope.Info("checking for existing launch template") - launchTemplate, launchTemplateUserDataHash, err := ec2svc.GetLaunchTemplate(machinePoolScope.Name()) - if err != nil { - conditions.MarkUnknown(machinePoolScope.AWSMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateNotFoundReason, err.Error()) - return err - } - - imageID, err := ec2svc.DiscoverLaunchTemplateAMI(machinePoolScope) - if err != nil { - conditions.MarkFalse(machinePoolScope.AWSMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateCreateFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return err - } - - if launchTemplate == nil { - machinePoolScope.Info("no existing launch template found, creating") - launchTemplateID, err := ec2svc.CreateLaunchTemplate(machinePoolScope, imageID, bootstrapData) - if err != nil { - conditions.MarkFalse(machinePoolScope.AWSMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateCreateFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return err - } - - machinePoolScope.SetLaunchTemplateIDStatus(launchTemplateID) - return machinePoolScope.PatchObject() - } - - // LaunchTemplateID is set during LaunchTemplate creation, but for a scenario such as `clusterctl move`, status fields become blank. - // If launchTemplate already exists but LaunchTemplateID field in the status is empty, get the ID and update the status. - if machinePoolScope.AWSMachinePool.Status.LaunchTemplateID == "" { - launchTemplateID, err := ec2svc.GetLaunchTemplateID(machinePoolScope.Name()) - if err != nil { - conditions.MarkUnknown(machinePoolScope.AWSMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateNotFoundReason, err.Error()) - return err - } - machinePoolScope.SetLaunchTemplateIDStatus(launchTemplateID) - return machinePoolScope.PatchObject() - } - - annotation, err := r.machinePoolAnnotationJSON(machinePoolScope.AWSMachinePool, TagsLastAppliedAnnotation) - if err != nil { - return err - } - - // Check if the instance tags were changed. If they were, create a new LaunchTemplate. - tagsChanged, _, _, _ := tagsChanged(annotation, machinePoolScope.AdditionalTags()) // nolint:dogsled - - needsUpdate, err := ec2svc.LaunchTemplateNeedsUpdate(machinePoolScope, &machinePoolScope.AWSMachinePool.Spec.AWSLaunchTemplate, launchTemplate) - if err != nil { - return err - } - - // If there is a change: before changing the template, check if there exist an ongoing instance refresh, - // because only 1 instance refresh can be "InProgress". If template is updated when refresh cannot be started, - // that change will not trigger a refresh. Do not start an instance refresh if only userdata changed. - if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID { - asgSvc := r.getASGService(ec2Scope) - canStart, err := asgSvc.CanStartASGInstanceRefresh(machinePoolScope) - if err != nil { - return err - } - if !canStart { - conditions.MarkFalse(machinePoolScope.AWSMachinePool, expinfrav1.InstanceRefreshStartedCondition, expinfrav1.InstanceRefreshNotReadyReason, clusterv1.ConditionSeverityWarning, "") - return errors.New("Cannot start a new instance refresh. Unfinished instance refresh exist") - } - } - - // Create a new launch template version if there's a difference in configuration, tags, - // userdata, OR we've discovered a new AMI ID. - if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID || launchTemplateUserDataHash != bootstrapDataHash { - machinePoolScope.Info("creating new version for launch template", "existing", launchTemplate, "incoming", machinePoolScope.AWSMachinePool.Spec.AWSLaunchTemplate) - // There is a limit to the number of Launch Template Versions. - // We ensure that the number of versions does not grow without bound by following a simple rule: Before we create a new version, we delete one old version, if there is at least one old version that is not in use. - if err := ec2svc.PruneLaunchTemplateVersions(machinePoolScope.AWSMachinePool.Status.LaunchTemplateID); err != nil { - return err - } - if err := ec2svc.CreateLaunchTemplateVersion(machinePoolScope, imageID, bootstrapData); err != nil { - return err - } - } - - // After creating a new version of launch template, instance refresh is required - // to trigger a rolling replacement of all previously launched instances. - // If ONLY the userdata changed, previously launched instances continue to use the old launch - // template. - // - // FIXME(dlipovetsky,sedefsavas): If the controller terminates, or the StartASGInstanceRefresh returns an error, - // this conditional will not evaluate to true the next reconcile. If any machines use an older - // Launch Template version, and the difference between the older and current versions is _more_ - // than userdata, we should start an Instance Refresh. - if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID { - machinePoolScope.Info("starting instance refresh", "number of instances", machinePoolScope.MachinePool.Spec.Replicas) - asgSvc := r.getASGService(ec2Scope) - if err := asgSvc.StartASGInstanceRefresh(machinePoolScope); err != nil { - conditions.MarkFalse(machinePoolScope.AWSMachinePool, expinfrav1.InstanceRefreshStartedCondition, expinfrav1.InstanceRefreshFailedReason, clusterv1.ConditionSeverityError, err.Error()) - return err - } - conditions.MarkTrue(machinePoolScope.AWSMachinePool, expinfrav1.InstanceRefreshStartedCondition) - } - - return nil -} - -func (r *AWSMachinePoolReconciler) reconcileTags(machinePoolScope *scope.MachinePoolScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope) error { - ec2Svc := r.getEC2Service(ec2Scope) - asgSvc := r.getASGService(clusterScope) - - launchTemplateID := machinePoolScope.AWSMachinePool.Status.LaunchTemplateID - asgName := machinePoolScope.Name() - additionalTags := machinePoolScope.AdditionalTags() - - tagsChanged, err := r.ensureTags(ec2Svc, asgSvc, machinePoolScope.AWSMachinePool, &launchTemplateID, &asgName, additionalTags) - if err != nil { - return err - } - if tagsChanged { - r.Recorder.Eventf(machinePoolScope.AWSMachinePool, corev1.EventTypeNormal, "UpdatedTags", "updated tags on resources") - } - return nil -} - // asgNeedsUpdates compares incoming AWSMachinePool and compares against existing ASG. func asgNeedsUpdates(machinePoolScope *scope.MachinePoolScope, existingASG *expinfrav1.AutoScalingGroup) bool { if machinePoolScope.MachinePool.Spec.Replicas != nil { diff --git a/exp/controllers/awsmachinepool_controller_test.go b/exp/controllers/awsmachinepool_controller_test.go index 138e022d45..78edab3c63 100644 --- a/exp/controllers/awsmachinepool_controller_test.go +++ b/exp/controllers/awsmachinepool_controller_test.go @@ -188,6 +188,8 @@ func TestAWSMachinePoolReconciler(t *testing.T) { defer teardown(t, g) getASG(t, g) + ec2Svc.EXPECT().ReconcileLaunchTemplate(gomock.Any(), gomock.Any(), gomock.Any()) + _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs) g.Expect(ms.AWSMachinePool.Finalizers).To(ContainElement(expinfrav1.MachinePoolFinalizer)) @@ -242,23 +244,6 @@ func TestAWSMachinePoolReconciler(t *testing.T) { setProviderID(t, g) expectedErr := errors.New("no connection available ") - var launchtemplate *expinfrav1.AWSLaunchTemplate - ec2Svc.EXPECT().GetLaunchTemplate(gomock.Any()).Return(launchtemplate, "", expectedErr) - _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs) - g.Expect(errors.Cause(err)).To(MatchError(expectedErr)) - }) - t.Run("should try to create a new machinepool if none exists", func(t *testing.T) { - g := NewWithT(t) - setup(t, g) - defer teardown(t, g) - setProviderID(t, g) - - expectedErr := errors.New("Invalid instance") - asgSvc.EXPECT().ASGIfExists(gomock.Any()).Return(nil, nil).AnyTimes() - ec2Svc.EXPECT().GetLaunchTemplate(gomock.Any()).Return(nil, "", nil) - ec2Svc.EXPECT().DiscoverLaunchTemplateAMI(gomock.Any()).Return(nil, nil) - ec2Svc.EXPECT().CreateLaunchTemplate(gomock.Any(), gomock.Any(), gomock.Any()).Return("", expectedErr).AnyTimes() - _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs) g.Expect(errors.Cause(err)).To(MatchError(expectedErr)) }) diff --git a/exp/controllers/awsmachinepool_tags.go b/exp/controllers/awsmachinepool_tags.go deleted file mode 100644 index 8a6a0ad9d8..0000000000 --- a/exp/controllers/awsmachinepool_tags.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "encoding/json" - - expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1" - "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services" -) - -const ( - // TagsLastAppliedAnnotation is the key for the AWSMachinePool object annotation - // which tracks the tags that the AWSMachinePool actuator is responsible - // for. These are the tags that have been handled by the - // AdditionalTags in the AWSMachinePool Provider Config. - // See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ - // for annotation formatting rules. - TagsLastAppliedAnnotation = "sigs.k8s.io/cluster-api-provider-aws-last-applied-tags" -) - -// Ensure that the tags of the AWSMachinePool are correct -// Returns bool, error -// Bool indicates if changes were made or not, allowing the caller to decide -// if the machine should be updated. -func (r *AWSMachinePoolReconciler) ensureTags(ec2svc services.EC2Interface, asgsvc services.ASGInterface, machinePool *expinfrav1.AWSMachinePool, launchTemplateID, asgName *string, additionalTags map[string]string) (bool, error) { - annotation, err := r.machinePoolAnnotationJSON(machinePool, TagsLastAppliedAnnotation) - if err != nil { - return false, err - } - - // Check if the instance tags were changed. If they were, update them. - // It would be possible here to only send new/updated tags, but for the - // moment we send everything, even if only a single tag was created or - // upated. - changed, created, deleted, newAnnotation := tagsChanged(annotation, additionalTags) - if changed { - err = ec2svc.UpdateResourceTags(launchTemplateID, created, deleted) - if err != nil { - return false, err - } - - if err := asgsvc.UpdateResourceTags(asgName, created, deleted); err != nil { - return false, err - } - - // We also need to update the annotation if anything changed. - err = r.updateMachinePoolAnnotationJSON(machinePool, TagsLastAppliedAnnotation, newAnnotation) - if err != nil { - return false, err - } - } - - return changed, nil -} - -// tagsChanged determines which tags to delete and which to add. -func tagsChanged(annotation map[string]interface{}, src map[string]string) (bool, map[string]string, map[string]string, map[string]interface{}) { - // Bool tracking if we found any changed state. - changed := false - - // Tracking for created/updated - created := map[string]string{} - - // Tracking for tags that were deleted. - deleted := map[string]string{} - - // The new annotation that we need to set if anything is created/updated. - newAnnotation := map[string]interface{}{} - - // Loop over annotation, checking if entries are in src. - // If an entry is present in annotation but not src, it has been deleted - // since last time. We flag this in the deleted map. - for t, v := range annotation { - _, ok := src[t] - - // Entry isn't in src, it has been deleted. - if !ok { - // Cast v to a string here. This should be fine, tags are always - // strings. - deleted[t] = v.(string) - changed = true - } - } - - // Loop over src, checking for entries in annotation. - // - // If an entry is in src, but not annotation, it has been created since - // last time. - // - // If an entry is in both src and annotation, we compare their values, if - // the value in src differs from that in annotation, the tag has been - // updated since last time. - for t, v := range src { - av, ok := annotation[t] - - // Entries in the src always need to be noted in the newAnnotation. We - // know they're going to be created or updated. - newAnnotation[t] = v - - // Entry isn't in annotation, it's new. - if !ok { - created[t] = v - newAnnotation[t] = v - changed = true - continue - } - - // Entry is in annotation, has the value changed? - if v != av { - created[t] = v - changed = true - } - - // Entry existed in both src and annotation, and their values were - // equal. Nothing to do. - } - - // We made it through the loop, and everything that was in src, was also - // in dst. Nothing changed. - return changed, created, deleted, newAnnotation -} - -// updateMachinePoolAnnotationJSON updates the `annotation` on `machinePool` with -// `content`. `content` in this case should be a `map[string]interface{}` -// suitable for turning into JSON. This `content` map will be marshalled into a -// JSON string before being set as the given `annotation`. -func (r *AWSMachinePoolReconciler) updateMachinePoolAnnotationJSON(machinePool *expinfrav1.AWSMachinePool, annotation string, content map[string]interface{}) error { - b, err := json.Marshal(content) - if err != nil { - return err - } - - r.updateMachinePoolAnnotation(machinePool, annotation, string(b)) - return nil -} - -// updateMachinePoolAnnotation updates the `annotation` on the given `machinePool` with -// `content`. -func (r *AWSMachinePoolReconciler) updateMachinePoolAnnotation(machinePool *expinfrav1.AWSMachinePool, annotation, content string) { - // Get the annotations - annotations := machinePool.GetAnnotations() - - if annotations == nil { - annotations = make(map[string]string) - } - - // Set our annotation to the given content. - annotations[annotation] = content - - // Update the machine object with these annotations - machinePool.SetAnnotations(annotations) -} - -// Returns a map[string]interface from a JSON annotation. -// This method gets the given `annotation` from the `machinePool` and unmarshalls it -// from a JSON string into a `map[string]interface{}`. -func (r *AWSMachinePoolReconciler) machinePoolAnnotationJSON(machinePool *expinfrav1.AWSMachinePool, annotation string) (map[string]interface{}, error) { - out := map[string]interface{}{} - - jsonAnnotation := r.machinePoolAnnotation(machinePool, annotation) - if len(jsonAnnotation) == 0 { - return out, nil - } - - err := json.Unmarshal([]byte(jsonAnnotation), &out) - if err != nil { - return out, err - } - - return out, nil -} - -// Fetches the specific machine annotation. -func (r *AWSMachinePoolReconciler) machinePoolAnnotation(machinePool *expinfrav1.AWSMachinePool, annotation string) string { - return machinePool.GetAnnotations()[annotation] -} diff --git a/exp/controllers/awsmanagedmachinepool_controller.go b/exp/controllers/awsmanagedmachinepool_controller.go index 50f5ebe9b8..fdb836f8dd 100644 --- a/exp/controllers/awsmanagedmachinepool_controller.go +++ b/exp/controllers/awsmanagedmachinepool_controller.go @@ -19,6 +19,9 @@ package controllers import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ec2" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -134,6 +137,17 @@ func (r *AWSManagedMachinePoolReconciler) Reconcile(ctx context.Context, req ctr return reconcile.Result{}, nil } + managedControlPlaneScope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{ + Client: r.Client, + Logger: &log, + Cluster: cluster, + ControlPlane: controlPlane, + ControllerName: "awsManagedControlPlane", + }) + if err != nil { + return ctrl.Result{}, errors.New("error getting managed control plane scope") + } + if !controlPlane.Status.Ready { log.Info("Control plane is not ready yet") conditions.MarkFalse(awsPool, expinfrav1.EKSNodegroupReadyCondition, expinfrav1.WaitingForEKSControlPlaneReason, clusterv1.ConditionSeverityInfo, "") @@ -150,6 +164,7 @@ func (r *AWSManagedMachinePoolReconciler) Reconcile(ctx context.Context, req ctr EnableIAM: r.EnableIAM, AllowAdditionalRoles: r.AllowAdditionalRoles, Endpoints: r.Endpoints, + InfraCluster: managedControlPlaneScope, }) if err != nil { return ctrl.Result{}, errors.Wrap(err, "failed to create scope") @@ -159,6 +174,7 @@ func (r *AWSManagedMachinePoolReconciler) Reconcile(ctx context.Context, req ctr applicableConditions := []clusterv1.ConditionType{ expinfrav1.EKSNodegroupReadyCondition, expinfrav1.IAMNodegroupRolesReadyCondition, + expinfrav1.LaunchTemplateReadyCondition, } conditions.SetSummary(machinePoolScope.ManagedMachinePool, conditions.WithConditions(applicableConditions...), conditions.WithStepCounter()) @@ -169,15 +185,16 @@ func (r *AWSManagedMachinePoolReconciler) Reconcile(ctx context.Context, req ctr }() if !awsPool.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, machinePoolScope) + return r.reconcileDelete(ctx, machinePoolScope, managedControlPlaneScope) } - return r.reconcileNormal(ctx, machinePoolScope) + return r.reconcileNormal(ctx, machinePoolScope, managedControlPlaneScope) } func (r *AWSManagedMachinePoolReconciler) reconcileNormal( _ context.Context, machinePoolScope *scope.ManagedMachinePoolScope, + ec2Scope scope.EC2Scope, ) (ctrl.Result, error) { machinePoolScope.Info("Reconciling AWSManagedMachinePool") @@ -187,6 +204,34 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal( } ekssvc := eks.NewNodegroupService(machinePoolScope) + ec2svc := r.getEC2Service(ec2Scope) + + if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil { + canUpdateLaunchTemplate := func() (bool, error) { + return true, nil + } + runPostLaunchTemplateUpdateOperation := func() error { + return nil + } + if err := ec2svc.ReconcileLaunchTemplate(machinePoolScope, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil { + r.Recorder.Eventf(machinePoolScope.ManagedMachinePool, corev1.EventTypeWarning, "FailedLaunchTemplateReconcile", "Failed to reconcile launch template: %v", err) + machinePoolScope.Error(err, "failed to reconcile launch template") + conditions.MarkFalse(machinePoolScope.ManagedMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateReconcileFailedReason, clusterv1.ConditionSeverityError, "") + return ctrl.Result{}, err + } + + launchTemplateID := machinePoolScope.GetLaunchTemplateIDStatus() + resourceServiceToUpdate := []scope.ResourceServiceToUpdate{{ + ResourceID: &launchTemplateID, + ResourceService: ec2svc, + }} + if err := ec2svc.ReconcileTags(machinePoolScope, resourceServiceToUpdate); err != nil { + return ctrl.Result{}, errors.Wrap(err, "error updating tags") + } + + // set the LaunchTemplateReady condition + conditions.MarkTrue(machinePoolScope.ManagedMachinePool, expinfrav1.LaunchTemplateReadyCondition) + } if err := ekssvc.ReconcilePool(); err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile machine pool for AWSManagedMachinePool %s/%s", machinePoolScope.ManagedMachinePool.Namespace, machinePoolScope.ManagedMachinePool.Name) @@ -198,15 +243,45 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal( func (r *AWSManagedMachinePoolReconciler) reconcileDelete( _ context.Context, machinePoolScope *scope.ManagedMachinePoolScope, + ec2Scope scope.EC2Scope, ) (ctrl.Result, error) { machinePoolScope.Info("Reconciling deletion of AWSManagedMachinePool") ekssvc := eks.NewNodegroupService(machinePoolScope) + ec2Svc := ec2.NewService(ec2Scope) if err := ekssvc.ReconcilePoolDelete(); err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile machine pool deletion for AWSManagedMachinePool %s/%s", machinePoolScope.ManagedMachinePool.Namespace, machinePoolScope.ManagedMachinePool.Name) } + if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil { + launchTemplateID := machinePoolScope.ManagedMachinePool.Status.LaunchTemplateID + launchTemplate, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName()) + if err != nil { + return ctrl.Result{}, err + } + + if launchTemplateID == nil { + id, _ := ec2Svc.GetLaunchTemplateID(machinePoolScope.LaunchTemplateName()) + launchTemplateID = &id + } + + if launchTemplate == nil { + machinePoolScope.V(2).Info("Unable to locate launch template") + r.Recorder.Eventf(machinePoolScope.ManagedMachinePool, corev1.EventTypeNormal, "NoLaunchTemplateFound", "Unable to find matching launch template") + controllerutil.RemoveFinalizer(machinePoolScope.ManagedMachinePool, expinfrav1.ManagedMachinePoolFinalizer) + return ctrl.Result{}, nil + } + + machinePoolScope.Info("deleting launch template", "name", launchTemplate.Name) + if err := ec2Svc.DeleteLaunchTemplate(*launchTemplateID); err != nil { + r.Recorder.Eventf(machinePoolScope.ManagedMachinePool, corev1.EventTypeWarning, "FailedDelete", "Failed to delete launch template %q: %v", launchTemplate.Name, err) + return ctrl.Result{}, errors.Wrap(err, "failed to delete launch template") + } + + machinePoolScope.Info("successfully deleted launch template") + } + controllerutil.RemoveFinalizer(machinePoolScope.ManagedMachinePool, expinfrav1.ManagedMachinePoolFinalizer) return reconcile.Result{}, nil @@ -272,3 +347,7 @@ func managedControlPlaneToManagedMachinePoolMapFunc(c client.Client, gvk schema. return results } } + +func (r *AWSManagedMachinePoolReconciler) getEC2Service(scope scope.EC2Scope) services.EC2Interface { + return ec2.NewService(scope) +} diff --git a/go.mod b/go.mod index 972163b320..89281bf1c8 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,18 @@ module sigs.k8s.io/cluster-api-provider-aws -go 1.17 +go 1.19 -replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.1.5 +replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.2.0 require ( github.com/alessio/shellescape v1.4.1 github.com/apparentlymart/go-cidr v1.1.0 - github.com/aws/amazon-vpc-cni-k8s v1.11.2 - github.com/aws/aws-lambda-go v1.32.0 - github.com/aws/aws-sdk-go v1.43.29 + github.com/aws/amazon-vpc-cni-k8s v1.10.2 + github.com/aws/aws-lambda-go v1.28.0 + github.com/aws/aws-sdk-go v1.43.28 github.com/awslabs/goformation/v4 v4.19.5 github.com/blang/semver v3.5.1+incompatible + github.com/cert-manager/cert-manager v1.8.2 github.com/flatcar-linux/ignition v0.36.1 github.com/go-logr/logr v1.2.3 github.com/gofrs/flock v0.8.1 @@ -26,131 +27,140 @@ require ( github.com/sergi/go-diff v1.2.0 github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/text v0.3.7 + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 + golang.org/x/text v0.7.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.23.5 - k8s.io/apiextensions-apiserver v0.23.5 - k8s.io/apimachinery v0.23.5 - k8s.io/cli-runtime v0.23.0 - k8s.io/client-go v0.23.5 - k8s.io/component-base v0.23.5 - k8s.io/klog/v2 v2.60.1 + k8s.io/api v0.24.2 + k8s.io/apiextensions-apiserver v0.24.2 + k8s.io/apimachinery v0.24.2 + k8s.io/cli-runtime v0.24.0 + k8s.io/client-go v0.24.2 + k8s.io/component-base v0.24.2 + k8s.io/klog/v2 v2.70.1 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 sigs.k8s.io/aws-iam-authenticator v0.5.9 - sigs.k8s.io/cluster-api v1.1.5 - sigs.k8s.io/cluster-api/test v1.1.5 - sigs.k8s.io/controller-runtime v0.11.2 + sigs.k8s.io/cluster-api v1.2.0 + sigs.k8s.io/cluster-api/test v1.2.0 + sigs.k8s.io/controller-runtime v0.12.3 + sigs.k8s.io/kustomize/api v0.11.4 sigs.k8s.io/yaml v1.3.0 ) require ( - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/BurntSushi/toml v1.0.0 // indirect - github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.2 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/BurntSushi/toml v1.0.0 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coredns/caddy v1.1.0 // indirect - github.com/coredns/corefile-migration v1.0.17 // indirect + github.com/coredns/corefile-migration v1.0.18 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.16+incompatible // indirect + github.com/docker/docker v20.10.17+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect + github.com/emicklei/go-restful v2.16.0+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/go-errors/errors v1.0.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect - github.com/gobuffalo/flect v0.2.4 // indirect + github.com/gobuffalo/flect v0.2.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/cel-go v0.9.0 // indirect - github.com/google/go-github/v33 v33.0.0 // indirect - github.com/google/go-querystring v1.0.0 // indirect + github.com/google/cel-go v0.10.1 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-github/v45 v45.2.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect - github.com/google/uuid v1.2.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect - github.com/gosuri/uitable v0.0.4 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/spdystream v0.2.0 // indirect - github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b // indirect github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 // indirect - github.com/shopspring/decimal v1.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.10.0 // indirect + github.com/spf13/viper v1.12.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.3.0 // indirect github.com/valyala/fastjson v1.6.3 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect - google.golang.org/grpc v1.42.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect + google.golang.org/grpc v1.47.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiserver v0.23.5 // indirect - k8s.io/cluster-bootstrap v0.23.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/kubectl v0.23.0 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiserver v0.24.2 // indirect + k8s.io/cluster-bootstrap v0.24.0 // indirect + k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661 // indirect + k8s.io/kubectl v0.24.0 // indirect + sigs.k8s.io/gateway-api v0.4.1 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/kind v0.14.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) diff --git a/go.sum b/go.sum index ad82a0c99a..8e6e529986 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,18 +16,10 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -36,8 +29,6 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -47,19 +38,18 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= @@ -68,7 +58,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= @@ -78,10 +67,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -89,38 +76,35 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ahmetb/gen-crd-api-reference-docs v0.3.0/go.mod h1:TdjdkYhlOifCQWPs1UdTma97kQQMozf5h26hTuG70u8= github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/amazon-vpc-cni-k8s v1.11.2 h1:0jVBhEywahqKDJ2rt8oX08qOPahYWaUXEDxuTx1RD9k= -github.com/aws/amazon-vpc-cni-k8s v1.11.2/go.mod h1:7bgoYaMokxHRLDMW1snJwDUa6lU2tNFSs+1OztRYU10= -github.com/aws/aws-lambda-go v1.32.0 h1:i8MflawW1hoyYp85GMH7LhvAs4cqzL7LOS6fSv8l2KM= -github.com/aws/aws-lambda-go v1.32.0/go.mod h1:IF5Q7wj4VyZyUFnZ54IQqeWtctHQ9tz+KhcbDenr220= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/aws/amazon-vpc-cni-k8s v1.10.2 h1:fT5lGj9A1xEe0vVZvRvaOzUoPONatn/ScUn3aGrrCvc= +github.com/aws/amazon-vpc-cni-k8s v1.10.2/go.mod h1:gYX+C9AK7x0VJ/GNOn5+6QO/DHuhLA4dZuupwZQyJc4= +github.com/aws/aws-lambda-go v1.28.0 h1:fZiik1PZqW2IyAN4rj+Y0UBaO1IDFlsNo9Zz/XnArK4= +github.com/aws/aws-lambda-go v1.28.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= +github.com/aws/aws-sdk-go v1.43.28 h1:HrBUf2pYEMRB3GDkSa/bZ2lkZIe8gSUOz/IEupG1Te0= github.com/aws/aws-sdk-go v1.43.28/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.43.29 h1:P6tBpMLwVLS/QwPkaBxfDIF3SmPouoacIk+/7NKnDxY= -github.com/aws/aws-sdk-go v1.43.29/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/awslabs/goformation/v4 v4.19.5 h1:Y+Tzh01tWg8gf//AgGKUamaja7Wx9NPiJf1FpZu4/iU= github.com/awslabs/goformation/v4 v4.19.5/go.mod h1:JoNpnVCBOUtEz9bFxc9sjy8uBUCLF5c4D1L7RhRTVM8= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= @@ -134,12 +118,12 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cert-manager/cert-manager v1.8.2 h1:cqPar+74IDp5dQxT9Icp1eThfca19Ml3RfNh8pDY7dI= +github.com/cert-manager/cert-manager v1.8.2/go.mod h1:95Ds29nFWH6YqEgLiQ9WTtsDnTcxrkUPRNfYaKVOzeM= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -148,34 +132,25 @@ github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg= github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0= github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= -github.com/coredns/corefile-migration v1.0.17 h1:tNwh8+4WOANV6NjSljwgW7qViJfhvPUt1kosj4rR8yg= -github.com/coredns/corefile-migration v1.0.17/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= +github.com/coredns/corefile-migration v1.0.18 h1:zs5PJm/VGZVje1ESRj6ZqyUuVsVfagExkbLU2QKV5mI= +github.com/coredns/corefile-migration v1.0.18/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -189,6 +164,7 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -197,26 +173,20 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ= -github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= +github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= @@ -226,6 +196,8 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7fo github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= +github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -233,12 +205,9 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= @@ -248,25 +217,25 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.2/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -275,17 +244,18 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -297,15 +267,16 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9 github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I= -github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/gobuffalo/flect v0.2.5 h1:H6vvsv2an0lalEaCDRThvtBfmg44W/QHXBCYUXf/6S4= +github.com/gobuffalo/flect v0.2.5/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -313,7 +284,6 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -355,14 +325,15 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.9.0 h1:u1hg7lcZ/XWw2d3aV1jFS30ijQQ6q0/h1C2ZBeBD1gY= -github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-go v0.10.1 h1:MQBGSZGnDwh7T/un+mzGKOMz3x+4E/GDPprWjDL+1Jg= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -377,11 +348,10 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= -github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= -github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= +github.com/google/go-github/v45 v45.2.0/go.mod h1:FObaZJEDSTa/WGCzZ2Z3eoCDXWJKMenWWTrd8jrta28= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f h1:7MmqygqdeJtziBUpm4Z9ThROFZUaVGaePMfcDnluf1E= github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f/go.mod h1:n1ej5+FqyEytMt/mugVDZLIiqTMO+vsrgY+kM6ohzN0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -393,7 +363,6 @@ github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f/go.mod h1:nOFQdrUlIl github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -403,34 +372,26 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= -github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -440,24 +401,13 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -470,29 +420,21 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -506,7 +448,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -526,6 +467,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -534,11 +476,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -546,16 +487,12 @@ github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -564,15 +501,11 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -586,17 +519,17 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -606,8 +539,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -627,21 +560,17 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -650,12 +579,14 @@ github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrB github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= +github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -663,15 +594,14 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= @@ -679,42 +609,39 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE= -github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b h1:jUK33OXuZP/l6babJtnLo1qsGvq6G9so9KMflGAm4YA= github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b/go.mod h1:8458kAagoME2+LN5//WxE71ysZ3B7r22fdgb7qVmXSY= github.com/sanathkr/yaml v0.0.0-20170819201035-0056894fa522 h1:fOCp11H0yuyAt2wqlbJtbyPzSgaxHTv8uN1pMpkG1t8= @@ -728,7 +655,6 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -744,15 +670,14 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= +github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= @@ -767,12 +692,10 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= -github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= +github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -783,22 +706,21 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= +github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -806,13 +728,11 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/telnet v0.0.0-20180329124119-c3b780dc415b/go.mod h1:IZpXDfkJ6tWD3PhBK5YzgQT+xJWh7OsdwiG8hA2MkO4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -825,7 +745,6 @@ go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQc go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= @@ -853,27 +772,22 @@ go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqe go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= +go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -881,16 +795,18 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -923,12 +839,11 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -943,7 +858,6 @@ golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -951,8 +865,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -968,29 +880,27 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1004,12 +914,10 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1030,7 +938,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1042,12 +949,9 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1056,10 +960,8 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1071,14 +973,11 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1086,42 +985,35 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1131,16 +1023,17 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1159,11 +1052,8 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1188,7 +1078,6 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1201,20 +1090,16 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1239,17 +1124,6 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1297,47 +1171,27 @@ google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I= +google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= @@ -1349,15 +1203,11 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1370,9 +1220,10 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1382,14 +1233,12 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= +gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1407,10 +1256,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -1421,95 +1269,101 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +inet.af/netaddr v0.0.0-20220617031823-097006376321 h1:B4dC8ySKTQXasnjDTMsoCMf1sQG4WsMej0WXaHxunmU= +k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= -k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= -k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= +k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= +k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= +k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= +k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= +k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= -k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/cli-runtime v0.23.0 h1:UONt0BV2+edjUVAXuR1nnOAL2CB9r+Gl9yk4UBQpKfs= -k8s.io/cli-runtime v0.23.0/go.mod h1:B5N3YH0KP1iKr6gEuJ/RRmGjO0mJQ/f/JrsmEiPQAlU= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= +k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= +k8s.io/apiserver v0.24.2 h1:orxipm5elPJSkkFNlwH9ClqaKEDJJA3yR2cAAlCnyj4= +k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= +k8s.io/cli-runtime v0.24.0 h1:ot3Qf49T852uEyNApABO1UHHpFIckKK/NqpheZYN2gM= +k8s.io/cli-runtime v0.24.0/go.mod h1:9XxoZDsEkRFUThnwqNviqzljtT/LdHtNWvcNFrAXl0A= +k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= -k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= -k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/cluster-bootstrap v0.23.0 h1:8pZuuAWPoygewSNB4IddX3HBwXcQkPDXL/ca7GtGf4o= -k8s.io/cluster-bootstrap v0.23.0/go.mod h1:VltEnKWfrRTiKgOXp3ts3vh7yqNlH6KFKFflo9GtCBg= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= +k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= +k8s.io/cluster-bootstrap v0.24.0 h1:MTs2x3Vfcl/PWvB5bfX7gzTFRyi4ZSbNSQgGJTCb6Sw= +k8s.io/cluster-bootstrap v0.24.0/go.mod h1:xw+IfoaUweMCAoi+VYhmqkcjii2G7gNg59dmGn7hi0g= +k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= +k8s.io/code-generator v0.22.0/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/code-generator v0.22.1/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= +k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= -k8s.io/component-base v0.23.0/go.mod h1:DHH5uiFvLC1edCpvcTDV++NKULdYYU6pR9Tt3HIKMKI= -k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= -k8s.io/component-helpers v0.23.0/go.mod h1:liXMh6FZS4qamKtMJQ7uLHnFe3tlC86RX5mJEk/aerg= -k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28/go.mod h1:9a7E6pmKLfuq8ZL31k2PDpgvSdyZfUOH9czlEmpblFk= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= +k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= +k8s.io/component-helpers v0.24.0/go.mod h1:Q2SlLm4h6g6lPTC9GMMfzdywfLSvJT2f1hOnnjaWD8c= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20201203183100-97869a43a9d9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kubectl v0.23.0 h1:WABWfj+Z4tC3SfKBCtZr5sIVHsFtkU9Azii4DR9IT6Y= -k8s.io/kubectl v0.23.0/go.mod h1:TfcGEs3u4dkmoC2eku1GYymdGaMtPMcaLLFrX/RB2kI= -k8s.io/metrics v0.23.0/go.mod h1:NDiZTwppEtAuKJ1Rxt3S4dhyRzdp6yUcJf0vo023dPo= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661 h1:nqYOUleKLC/0P1zbU29F5q6aoezM6MOAVz+iyfQbZ5M= +k8s.io/kube-openapi v0.0.0-20220401212409-b28bf2818661/go.mod h1:daOouuuwd9JXpv1L7Y34iV3yf6nxzipkKMWWlqlvK9M= +k8s.io/kubectl v0.24.0 h1:nA+WtMLVdXUs4wLogGd1mPTAesnLdBpCVgCmz3I7dXo= +k8s.io/kubectl v0.24.0/go.mod h1:pdXkmCyHiRTqjYfyUJiXtbVNURhv0/Q1TyRhy2d5ic0= +k8s.io/metrics v0.24.0/go.mod h1:jrLlFGdKl3X+szubOXPG0Lf2aVxuV3QJcbsgVRAM6fI= k8s.io/sample-controller v0.22.1/go.mod h1:184Fa29md4PuQSEozdEw6n+AAmoodWOy9iCtyfCvAWY= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/aws-iam-authenticator v0.5.9 h1:r0xXKCAZVBKRZkJRdy/nm8bEQbBjjw+3xi0rzi31RJE= sigs.k8s.io/aws-iam-authenticator v0.5.9/go.mod h1:6dId2LCc8oHqeBzP6E8ndp4DflhKTxYLb5ZXwI4YmFA= -sigs.k8s.io/cluster-api v1.1.5 h1:4gV3mg1mlwrm3bFWeXADfL9ILRyufjR47KpwBL3PT6Y= -sigs.k8s.io/cluster-api v1.1.5/go.mod h1:WBPgw1yJdvybx/U3Ib49T9Q4JurairgnA/s0afxuE/w= -sigs.k8s.io/cluster-api/test v1.1.5 h1:jukMEA50R3V6G7VcJMR7dMZJPd3Tuf0Orc/6CKuS9Ek= -sigs.k8s.io/cluster-api/test v1.1.5/go.mod h1:NXFioUFKruk/PgpUt4QrprV9bN1rVUcm7OWao9dfesg= -sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= -sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/cluster-api v1.2.0 h1:jAKG78SQ6nUHfqbRD64FrYZR79P+J7/aiH17zbpsWcQ= +sigs.k8s.io/cluster-api v1.2.0/go.mod h1:oiuV+mlCV1QxDnuI+PfElFlAfuXHo9ZGVBojoihVtHY= +sigs.k8s.io/cluster-api/test v1.2.0 h1:l5rQajf3yg9kYbMTVpKX4y2j0SQixEdCqLx9ANrNvF8= +sigs.k8s.io/cluster-api/test v1.2.0/go.mod h1:JdMqpv9rEOFWQVQ8danpBduqxoQkZMiOvpIGJ7v8qjw= +sigs.k8s.io/controller-runtime v0.9.6/go.mod h1:q6PpkM5vqQubEKUKOM6qr06oXGzOBcCby1DA9FbyZeA= +sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio= +sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0= +sigs.k8s.io/controller-tools v0.6.2/go.mod h1:oaeGpjXn6+ZSEIQkUe/+3I40PNiDYp9aeawbt3xTgJ8= +sigs.k8s.io/gateway-api v0.4.1 h1:Tof9/PNSZXyfDuTTe1XFvaTlvBRE6bKq1kmV6jj6rQE= +sigs.k8s.io/gateway-api v0.4.1/go.mod h1:r3eiNP+0el+NTLwaTfOrCNXy8TukC+dIM3ggc+fbNWk= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= -sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8= -sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ= -sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io= -sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= +sigs.k8s.io/kustomize/api v0.11.4 h1:/0Mr3kfBBNcNPOW5Qwk/3eb8zkswCwnqQxxKtmrTkRo= +sigs.k8s.io/kustomize/api v0.11.4/go.mod h1:k+8RsqYbgpkIrJ4p9jcdPqe8DprLxFUUO0yNOq8C+xI= +sigs.k8s.io/kustomize/cmd/config v0.10.6/go.mod h1:/S4A4nUANUa4bZJ/Edt7ZQTyKOY9WCER0uBS1SW2Rco= +sigs.k8s.io/kustomize/kustomize/v4 v4.5.4/go.mod h1:Zo/Xc5FKD6sHl0lilbrieeGeZHVYCA4BzxeAaLI05Bg= +sigs.k8s.io/kustomize/kyaml v0.13.6/go.mod h1:yHP031rn1QX1lr/Xd934Ri/xdVNG8BE2ECa78Ht/kEg= +sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= diff --git a/kubeconfig.yaml b/kubeconfig.yaml new file mode 100644 index 0000000000..39e2300eaf --- /dev/null +++ b/kubeconfig.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMvakNDQWVhZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJek1ESXlPREEwTkRFMU5Wb1hEVE16TURJeU5UQTBOREUxTlZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBS2dNCk1ySGI5Y3dhQW1PYW0zelFjS0N3eVh6ckdvOUVJY3hFemllczE2VUNMdzNaQ0RqWFUyWGlQYm1qakNyU3FrYkgKdHFZa0d2RzVXOU0xVTBqN2pndkNVNXlHNWIzbzQ0eUVWZ0hyYWhldnE0MnJUc01xTXZ0b2txblhsUndad2U3aApsVENKZXJadUdVY3VMeGY4REMyMm9OeUJ0RXVXMUJaNDBwSk9OTUNGZU1vb0EyT3VnK05HUlk1K2ZTTXdlcGhXCmN5UFlBeGdDL2NRMHhDczZmeHY1UmEzT1R0VHR1bTN5Vm9RWHNCUXVub2NGOXhjUUJYUXZvc3czNEJkWS9iMGUKYlh2MTFnM0xoUW1aT0RCSVpISVRDTjR4OE9oNHljU2luaG16TmNEVkRzZ3ZrZlhQbEtaNWdiZzFLMldLaVlaaQpXRTNWeTNoMmx4WEdodVhxa1BNQ0F3RUFBYU5aTUZjd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZNZG1MRjBHL2xrc0hibXRvdStWRTNCcFZvbW5NQlVHQTFVZEVRUU8KTUF5Q0NtdDFZbVZ5Ym1WMFpYTXdEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBREl3V2gxTXVZNHpBbEoySDIzQgpNUlF4WXRjM2JZMGJLWVVTT3g2V0pGdlJxWnlMTTZWRzlQd1Y4Sy9uNFdYTUhHR0Q3VktLWEVjZ1ZGSjNsaGd1CmgrV3poazJjeWp2Tkk4Znd2RVNOcXM0dnA1NUZzUFZTMGxhTEhmclpkMGxRblNjVHdTVkZmTWNvSi9ZMFk4TzYKSHBQd0JCK0VuNHpEcnQ2bktyYzlOMnBROS9UMU9MdEJWTU94UTlTc2gzY01CK29HcTM4OExxK2IvMFZXSXVlYQpCUFIyaEZiZzdGbEJqTVQyS1NuQ252WUlTK0JVc3hXWEwzT1lEcnN2R3EwMHNCWWF4c2oxNXYySlpzZSs0UnNRClVSTGhsUHd2bEkwY1YvRTlzeHdKM0o1c2VKVWNtaXRra0MrbEVmRStqeDQxOFdSNXd6TXo4ZXVteGt0TndKVTMKMXJZPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + server: https://127.0.0.1:51651 + name: kind-capa +contexts: +- context: + cluster: kind-capa + user: kind-capa + name: kind-capa +current-context: kind-capa +kind: Config +preferences: {} +users: +- name: kind-capa + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURJVENDQWdtZ0F3SUJBZ0lJQkZtMkpXbzZjdDh3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TXpBeU1qZ3dORFF4TlRWYUZ3MHlOREF5TWpnd05EUXhOVGRhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQTA4aHJjb2l1UzFWc3FmaGcKNXQ0OEpvRSs1SWlqZW56Zm5TcmtNZFVDTWxMMDBqV25RTytRYW92bGN1OXQ5MGQyNVRFMWptbEpWMWNaVVRVbwpPVTRBZnA3WUdUbzNpbjBYTG1SWmNCUkdxdlplU0xQTjBWdHZsQ21ndXBmZlRmQTRrVkhSa0x2MldtK3U4YkJVCjZPTVZHdGVKdFJIam1xVjR5aFkrVXdHSEtaak1IcmFQTlFiRzRnUUt5VjVjekV6NFh0bk5NZjN1eUVNLzVnREsKejZDeTNpaEhZVkhoSHZWTTZVNTVKWU1sMUd0OFUrL3p4TVJ3MHJnZHA0UHl6U2ZZQzR0UHkrWmF2anFTQ1dvSApyK2RXQ0p4bUpFSE8yYS96UEliaTlyRlBGTVduRXZsMkNiWnc5aUZPOEl6YXVxVGhUYjEzVVNld01OS2E0OWN1CktLUEVxd0lEQVFBQm8xWXdWREFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JUSFppeGRCdjVaTEIyNXJhTHZsUk53YVZhSgpwekFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBS29tZ29jQzFFWDlPdDg1bHBQWWNBYkdRdFVZMUFZV0c1a0wxClJ0Z3A4aWtLOW95K3VDcHdBbXZ1NE5pSy9jMDRVbDZRLzZQcmo1Ri9xZG4zdks4eHBZbEwrcStVcXhORWcrL2wKbTN2UjVWSGF6bDF4UXR0VStJakdUbzhTYURhWjQxV0xwOExWbUI4T1BDYmprcm4yckhYVGdLVFpCdVNJM05rNApVWHdhc1pKdTE2bC92aVJSNzdZZTlFZmRPd1RiaFhJUkxJTUxRbXkyZ0MvaHYycEx6THF2dVRSa0tKQ0pENW03Ci9mS2FtY2lGZEg3dmRMVVNYOHhBZmNBcS9jc3RXYW9Fa0N6UnVQUlFCU1grdFQrYWo1d3AvSVR5SERYSGppb3QKazdNU2l1WnNNeFZmcUtMTXgrMkJ5S0l6V1ZSNzJzak41VTViZFVZNHovdktWQVpuRmc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMDhocmNvaXVTMVZzcWZoZzV0NDhKb0UrNUlpamVuemZuU3JrTWRVQ01sTDAwalduClFPK1Fhb3ZsY3U5dDkwZDI1VEUxam1sSlYxY1pVVFVvT1U0QWZwN1lHVG8zaW4wWExtUlpjQlJHcXZaZVNMUE4KMFZ0dmxDbWd1cGZmVGZBNGtWSFJrTHYyV20rdThiQlU2T01WR3RlSnRSSGptcVY0eWhZK1V3R0hLWmpNSHJhUApOUWJHNGdRS3lWNWN6RXo0WHRuTk1mM3V5RU0vNWdES3o2Q3kzaWhIWVZIaEh2Vk02VTU1SllNbDFHdDhVKy96CnhNUncwcmdkcDRQeXpTZllDNHRQeStaYXZqcVNDV29IcitkV0NKeG1KRUhPMmEvelBJYmk5ckZQRk1XbkV2bDIKQ2JadzlpRk84SXphdXFUaFRiMTNVU2V3TU5LYTQ5Y3VLS1BFcXdJREFRQUJBb0lCQUNtaWVSMG5LYnk5cW9Ndgp4VDBkNjQxV1UxcnlVZEUrdzJkT2kxWVFua0h1ZFJydVdWYzJ3Wmx3cjBjTk12MHVHMzJyVXZBUVdzN0UzTUZPClJmNGVDNUxZLzNyV0t4MGF3bmZvV256aVA1SlVwT3grRExSZjJXdXpVMW5iaVpiMDV1b0t2N3hJZDRJSCtzaWcKOWRCaTlEODJxR2ZrZ0c1aW9ZQjVKditDaVdQdnZvUUFubmY0ZittQU1jMXlQalZQRXFldW5GV01ORk9tVlE0Ywp2SkVJVFdXMGhUaVlDRC9kN1U0c3FhVTNUczFWaXRjSXhXZC9lM3NuaHNLa29uN2ptcXE4QU1wVklrWW5nVzN4CjhVS29sMTRJTTNtRUtucDhXaHNnVVJsZWFTUVREcnBPUVB2VDBFbmVFTnB5ZTh0MEdRNEFxV084ZFY4ZDdqTlkKUFBSU2ZpRUNnWUVBNlFBZ2ZyNzlYbC8vWVBZNDdQaklJb2wyOEdsalgyMlZiU0VITmF1SGRUZ3ZhMm1UdFl4cApIMDhJSTQvUFFsYjd0Lytibm5rUUxLTlM2VTVod1hRN0hLN0x0U2luTW5HWXIwY0FWYWJoZUwwcHlHQ1hjd05TCkx4d2VDOGN0emZIV016MTZoL3NwS1FwL2VoMm9uTHZyOTJiMnYyaDhyelNBVnVsczRzZE5VVDBDZ1lFQTZMQWcKM1MrUFZPcWNGV2Z0end6Z2VZRWRLTmRnUTlwRXZTUjQ3dXFlVWFNU29iQWJucjBhMFpDS0RiTHVpeGpqdVBQWgpzSE8rWVdGOGlIa0VOTzd3TXJGQ3JHVDZGQ0EvZi9CZEoxVmRhTTZFSnF2b2x5ZXlLUk1XcDhod25rRnpmV3g4ClBlUXNqYXRKam1WTjB3N2J5MnRrUWJkdkluUlkvcm9sNytPeGZBY0NnWUFjVmRtZDBGZzd1K3p3dThEY2dMekcKQm5XaWN3NlVzbHp6c29xUDZsWkUyUnBiTTdHMDZHaDRXRUlqSVFBUmowWjViK3BhOFZvWU01eUhwdktZZGl3aQpKdGU3SmtCL0QvV0RpMXp0RGFocDZJZEdTUWppQWN0aTAwYU1KQ0xiNHNjcjRCUWdxVml1RndxOXJSR3VUZVE2CmFCa2wyZTBmMllDcUVTVWV4eUlOK1FLQmdIQXB6bjJ3V2xBZGswV0NScERpN1NYRXhqQ1V1Vk9iMkR5d1l4Y1AKdzJjb3kyZGpRNzBkL05wOTZQNDNmV05SaVJ2emQ4RGwyaVNJNElYTWgrbVlML0xQbllzS1ZiM3oyWWdyYmt2eQppMGRsSFRDaTkwZzltMVRQWGNGZERtUkt1b2dBRGxlck1Jck53WFJCZ3VVMUtXQUNFVjdZTkZzUWFJSEphMVlhCmxNaHJBb0dBUm40d2xzQVcvVk0vYjFid3E1cTNIcUp6TUgyemlJOUZpVlRCTndROTdyKzhlL2VaUG13Z2hiNUUKOWxDR0p5RmZkVkVxY0xKZFRDRWEzcWpiMUJZTmJiZ0M4R2ZTMmh1QVN5cVhzOGFiL2F6Y1lLYno1YWxjNG8wMQpuenB5NktpUmpnS1J1akYyVDFFU0VJb2hyU2xMUXYzbzZWYk9LSStxVG04dkE1ZDMzMnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/main.go b/main.go index e5d78403d8..8ce03ee305 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "os" "time" + v1certmanager "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" cgscheme "k8s.io/client-go/kubernetes/scheme" @@ -62,6 +63,8 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/record" "sigs.k8s.io/cluster-api-provider-aws/version" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1beta1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" ) @@ -86,6 +89,9 @@ func init() { _ = infrav1alpha4.AddToScheme(scheme) _ = expinfrav1alpha4.AddToScheme(scheme) _ = expinfrav1.AddToScheme(scheme) + _ = v1certmanager.AddToScheme(scheme) + _ = controlplanev1beta1.AddToScheme(scheme) + _ = bootstrapv1beta1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -181,93 +187,97 @@ func main() { os.Exit(1) } - if err = (&controllers.AWSMachineReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("AWSMachine"), - Recorder: mgr.GetEventRecorderFor("awsmachine-controller"), - Endpoints: AWSServiceEndpoints, - WatchFilterValue: watchFilterValue, - }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsMachineConcurrency, RecoverPanic: true}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AWSMachine") - os.Exit(1) - } - if err = (&controllers.AWSClusterReconciler{ - Client: mgr.GetClient(), - Recorder: mgr.GetEventRecorderFor("awscluster-controller"), - Endpoints: AWSServiceEndpoints, - WatchFilterValue: watchFilterValue, - ExternalResourceGC: externalResourceGC, - }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsClusterConcurrency, RecoverPanic: true}); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "AWSCluster") - os.Exit(1) - } - enableGates(ctx, mgr, AWSServiceEndpoints, externalResourceGC) - - if err = (&infrav1.AWSMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachineTemplate") - os.Exit(1) - } - if err = (&infrav1.AWSCluster{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSCluster") - os.Exit(1) - } - if err = (&infrav1.AWSClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterTemplate") - os.Exit(1) - } - if err = (&infrav1.AWSClusterControllerIdentity{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterControllerIdentity") - os.Exit(1) - } - if err = (&infrav1.AWSClusterRoleIdentity{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterRoleIdentity") - os.Exit(1) - } - if err = (&infrav1.AWSClusterStaticIdentity{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterStaticIdentity") - os.Exit(1) - } - if err = (&infrav1.AWSMachine{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachine") - os.Exit(1) - } - if feature.Gates.Enabled(feature.EKS) { - setupLog.Info("enabling EKS webhooks") - if err := (&ekscontrolplanev1.AWSManagedControlPlane{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSManagedControlPlane") + setupLog.Info("webhook port", "webhook-port", webhookPort) + if webhookPort == 0 { + if err = (&controllers.AWSMachineReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("AWSMachine"), + Recorder: mgr.GetEventRecorderFor("awsmachine-controller"), + Endpoints: AWSServiceEndpoints, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsMachineConcurrency, RecoverPanic: true}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AWSMachine") os.Exit(1) } - if feature.Gates.Enabled(feature.EKSFargate) { - if err = (&expinfrav1.AWSFargateProfile{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSFargateProfile") + if err = (&controllers.AWSClusterReconciler{ + Client: mgr.GetClient(), + Recorder: mgr.GetEventRecorderFor("awscluster-controller"), + Endpoints: AWSServiceEndpoints, + WatchFilterValue: watchFilterValue, + ExternalResourceGC: externalResourceGC, + }).SetupWithManager(ctx, mgr, controller.Options{MaxConcurrentReconciles: awsClusterConcurrency, RecoverPanic: true}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "AWSCluster") + os.Exit(1) + } + enableGates(ctx, mgr, AWSServiceEndpoints, externalResourceGC) + } else { + if err = (&infrav1.AWSMachineTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachineTemplate") + os.Exit(1) + } + if err = (&infrav1.AWSCluster{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSCluster") + os.Exit(1) + } + if err = (&infrav1.AWSClusterTemplate{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterTemplate") + os.Exit(1) + } + if err = (&infrav1.AWSClusterControllerIdentity{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterControllerIdentity") + os.Exit(1) + } + if err = (&infrav1.AWSClusterRoleIdentity{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterRoleIdentity") + os.Exit(1) + } + if err = (&infrav1.AWSClusterStaticIdentity{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSClusterStaticIdentity") + os.Exit(1) + } + if err = (&infrav1.AWSMachine{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachine") + os.Exit(1) + } + if feature.Gates.Enabled(feature.EKS) { + setupLog.Info("enabling EKS webhooks") + if err := (&ekscontrolplanev1.AWSManagedControlPlane{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSManagedControlPlane") os.Exit(1) } + if feature.Gates.Enabled(feature.EKSFargate) { + if err = (&expinfrav1.AWSFargateProfile{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSFargateProfile") + os.Exit(1) + } + } + if feature.Gates.Enabled(feature.MachinePool) { + if err = (&expinfrav1.AWSManagedMachinePool{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSManagedMachinePool") + os.Exit(1) + } + } } + if feature.Gates.Enabled(feature.MachinePool) { - if err = (&expinfrav1.AWSManagedMachinePool{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSManagedMachinePool") + setupLog.Info("enabling webhook for AWSMachinePool") + if err = (&expinfrav1.AWSMachinePool{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachinePool") os.Exit(1) } } - } - if feature.Gates.Enabled(feature.MachinePool) { - setupLog.Info("enabling webhook for AWSMachinePool") - if err = (&expinfrav1.AWSMachinePool{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "AWSMachinePool") - os.Exit(1) - } - } - // +kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder - if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create ready check") - os.Exit(1) - } + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } - if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { - setupLog.Error(err, "unable to create health check") - os.Exit(1) + if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create health check") + os.Exit(1) + } } setupLog.Info("starting manager", "version", version.Get().String()) @@ -447,7 +457,7 @@ func initFlags(fs *pflag.FlagSet) { fs.IntVar(&webhookPort, "webhook-port", - 9443, + 0, "Webhook Server port.", ) diff --git a/pkg/cloud/awserrors/errors.go b/pkg/cloud/awserrors/errors.go index 08ccf2bcb0..ceea023a2e 100644 --- a/pkg/cloud/awserrors/errors.go +++ b/pkg/cloud/awserrors/errors.go @@ -25,22 +25,23 @@ import ( // Error singletons for AWS errors. const ( - AssociationIDNotFound = "InvalidAssociationID.NotFound" - AuthFailure = "AuthFailure" - BucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" - EIPNotFound = "InvalidElasticIpID.NotFound" - GatewayNotFound = "InvalidGatewayID.NotFound" - GroupNotFound = "InvalidGroup.NotFound" - InternetGatewayNotFound = "InvalidInternetGatewayID.NotFound" - InUseIPAddress = "InvalidIPAddress.InUse" - InvalidAccessKeyID = "InvalidAccessKeyId" - InvalidClientTokenID = "InvalidClientTokenId" - InvalidInstanceID = "InvalidInstanceID.NotFound" - InvalidSubnet = "InvalidSubnet" - LaunchTemplateNameNotFound = "InvalidLaunchTemplateName.NotFoundException" - LoadBalancerNotFound = "LoadBalancerNotFound" - NATGatewayNotFound = "InvalidNatGatewayID.NotFound" - // nolint:gosec + AssociationIDNotFound = "InvalidAssociationID.NotFound" + AuthFailure = "AuthFailure" + BucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" + EIPNotFound = "InvalidElasticIpID.NotFound" + GatewayNotFound = "InvalidGatewayID.NotFound" + GroupNotFound = "InvalidGroup.NotFound" + InternetGatewayNotFound = "InvalidInternetGatewayID.NotFound" + EgressOnlyInternetGatewayNotFound = "InvalidEgressOnlyInternetGatewayID.NotFound" + InUseIPAddress = "InvalidIPAddress.InUse" + InvalidAccessKeyID = "InvalidAccessKeyId" + InvalidClientTokenID = "InvalidClientTokenId" + InvalidInstanceID = "InvalidInstanceID.NotFound" + InvalidSubnet = "InvalidSubnet" + LaunchTemplateNameNotFound = "InvalidLaunchTemplateName.NotFoundException" + LoadBalancerNotFound = "LoadBalancerNotFound" + NATGatewayNotFound = "InvalidNatGatewayID.NotFound" + //nolint:gosec NoCredentialProviders = "NoCredentialProviders" NoSuchKey = "NoSuchKey" PermissionNotFound = "InvalidPermission.NotFound" @@ -50,6 +51,7 @@ const ( SubnetNotFound = "InvalidSubnetID.NotFound" UnrecognizedClientException = "UnrecognizedClientException" VPCNotFound = "InvalidVpcID.NotFound" + VPCMissingParameter = "MissingParameter" ErrCodeRepositoryAlreadyExistsException = "RepositoryAlreadyExistsException" ) diff --git a/pkg/cloud/converters/eks.go b/pkg/cloud/converters/eks.go index 53e79a84fe..ed790f3b82 100644 --- a/pkg/cloud/converters/eks.go +++ b/pkg/cloud/converters/eks.go @@ -148,16 +148,19 @@ func TaintEffectFromSDK(effect string) (expinfrav1.TaintEffect, error) { func ConvertSDKToIdentityProvider(in *ekscontrolplanev1.OIDCIdentityProviderConfig) *identityprovider.OidcIdentityProviderConfig { if in != nil { + if in.RequiredClaims == nil { + in.RequiredClaims = make(map[string]string) + } return &identityprovider.OidcIdentityProviderConfig{ ClientID: in.ClientID, - GroupsClaim: in.GroupsClaim, - GroupsPrefix: in.GroupsPrefix, + GroupsClaim: aws.StringValue(in.GroupsClaim), + GroupsPrefix: aws.StringValue(in.GroupsPrefix), IdentityProviderConfigName: in.IdentityProviderConfigName, IssuerURL: in.IssuerURL, - RequiredClaims: aws.StringMap(in.RequiredClaims), + RequiredClaims: in.RequiredClaims, Tags: in.Tags, - UsernameClaim: in.UsernameClaim, - UsernamePrefix: in.UsernamePrefix, + UsernameClaim: aws.StringValue(in.UsernameClaim), + UsernamePrefix: aws.StringValue(in.UsernamePrefix), } } diff --git a/pkg/cloud/converters/tags.go b/pkg/cloud/converters/tags.go index ec36b01f58..8bbb61f730 100644 --- a/pkg/cloud/converters/tags.go +++ b/pkg/cloud/converters/tags.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/secretsmanager" "github.com/aws/aws-sdk-go/service/ssm" @@ -124,6 +125,22 @@ func MapToSSMTags(src infrav1.Tags) []*ssm.Tag { return tags } +// MapToIAMTags converts a infrav1.Tags to a []*iam.Tag. +func MapToIAMTags(src infrav1.Tags) []*iam.Tag { + tags := make([]*iam.Tag, 0, len(src)) + + for k, v := range src { + tag := &iam.Tag{ + Key: aws.String(k), + Value: aws.String(v), + } + + tags = append(tags, tag) + } + + return tags +} + // ASGTagsToMap converts a []*autoscaling.TagDescription into a infrav1.Tags. func ASGTagsToMap(src []*autoscaling.TagDescription) infrav1.Tags { tags := make(infrav1.Tags, len(src)) diff --git a/pkg/cloud/scope/awsnode.go b/pkg/cloud/scope/awsnode.go index a1a74ede5d..ae49629357 100644 --- a/pkg/cloud/scope/awsnode.go +++ b/pkg/cloud/scope/awsnode.go @@ -17,6 +17,7 @@ limitations under the License. package scope import ( + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -37,4 +38,8 @@ type AWSNodeScope interface { SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup // DisableVPCCNI returns whether the AWS VPC CNI should be disabled DisableVPCCNI() bool + // VpcCni specifies configuration related to the VPC CNI. + VpcCni() ekscontrolplanev1.VpcCni + // VPC returns the given VPC configuration. + VPC() *infrav1.VPCSpec } diff --git a/pkg/cloud/scope/cluster.go b/pkg/cloud/scope/cluster.go index 02dea371ad..6370f73c3b 100644 --- a/pkg/cloud/scope/cluster.go +++ b/pkg/cloud/scope/cluster.go @@ -19,6 +19,9 @@ package scope import ( "context" "fmt" + "sigs.k8s.io/cluster-api-provider-aws/util/system" + + "sigs.k8s.io/cluster-api/controllers/remote" awsclient "github.com/aws/aws-sdk-go/aws/client" "github.com/go-logr/logr" @@ -34,6 +37,14 @@ import ( "sigs.k8s.io/cluster-api/util/patch" ) +const ( + // KubeconfigReadyAnnotation is the key for the cluster object annotation + // which tracks if the Kubeconfig is available to communicate with the workload cluster + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + // for annotation formatting rules. + KubeconfigReadyAnnotation = "sigs.k8s.io/cluster-api-provider-aws-kubeconfig-ready" +) + // ClusterScopeParams defines the input parameters used to create a new Scope. type ClusterScopeParams struct { Client client.Client @@ -62,7 +73,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { clusterScope := &ClusterScope{ Logger: *params.Logger, - client: params.Client, + Client: params.Client, Cluster: params.Cluster, AWSCluster: params.AWSCluster, controllerName: params.ControllerName, @@ -88,7 +99,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) { // ClusterScope defines the basic context for an actuator to operate upon. type ClusterScope struct { logr.Logger - client client.Client + Client client.Client patchHelper *patch.Helper Cluster *clusterv1.Cluster @@ -99,6 +110,27 @@ type ClusterScope struct { controllerName string } +// RemoteClient returns the Kubernetes Client for connecting to the workload cluster. +func (s *ClusterScope) RemoteClient() (client.Client, error) { + clusterKey := client.ObjectKey{ + Name: s.Name(), + Namespace: s.Namespace(), + } + + restConfig, err := remote.RESTConfig(context.Background(), s.Cluster.Name, s.Client, clusterKey) + if err != nil { + return nil, fmt.Errorf("getting remote rest config for %s/%s: %w", s.Namespace(), s.Name(), err) + } + restConfig.Timeout = DefaultKubeClientTimeout + + return client.New(restConfig, client.Options{Scheme: scheme}) +} + +// ManagementClient returns the Kubernetes Client for the management cluster. +func (s *ClusterScope) ManagementClient() client.Client { + return s.Client +} + // Network returns the cluster network object. func (s *ClusterScope) Network() *infrav1.NetworkStatus { return &s.AWSCluster.Status.Network @@ -234,6 +266,9 @@ func (s *ClusterScope) PatchObject() error { if s.AWSCluster.Spec.Bastion.Enabled { applicableConditions = append(applicableConditions, infrav1.BastionHostReadyCondition) } + if s.VPC().EnableIPv6 { + applicableConditions = append(applicableConditions, infrav1.EgressOnlyInternetGatewayReadyCondition) + } } conditions.SetSummary(s.AWSCluster, @@ -250,6 +285,7 @@ func (s *ClusterScope) PatchObject() error { infrav1.VpcReadyCondition, infrav1.SubnetsReadyCondition, infrav1.InternetGatewayReadyCondition, + infrav1.EgressOnlyInternetGatewayReadyCondition, infrav1.NatGatewaysReadyCondition, infrav1.RouteTablesReadyCondition, infrav1.ClusterSecurityGroupsReadyCondition, @@ -317,6 +353,20 @@ func (s *ClusterScope) Bastion() *infrav1.Bastion { return &s.AWSCluster.Spec.Bastion } +// S3Bucket returns the s3 bucket details. +func (s *ClusterScope) S3Bucket() *infrav1.S3Bucket { + return s.AWSCluster.Spec.S3Bucket +} + +// AssociateOIDCProvider returns if the cluster should have an OIDC Provider Associated. +func (s *ClusterScope) AssociateOIDCProvider() bool { + return s.AWSCluster.Spec.AssociateOIDCProvider +} + +func (s *ClusterScope) OIDCProviderStatus() *infrav1.OIDCProviderStatus { + return &s.AWSCluster.Status.OIDCProvider +} + // SetBastionInstance sets the bastion instance in the status of the cluster. func (s *ClusterScope) SetBastionInstance(instance *infrav1.Instance) { s.AWSCluster.Status.Bastion = instance @@ -347,3 +397,11 @@ func (s *ClusterScope) ImageLookupOrg() string { func (s *ClusterScope) ImageLookupBaseOS() string { return s.AWSCluster.Spec.ImageLookupBaseOS } + +// Partition returns the cluster partition. +func (s *ClusterScope) Partition() string { + if s.AWSCluster.Spec.Partition == "" { + s.AWSCluster.Spec.Partition = system.GetPartitionFromRegion(s.Region()) + } + return s.AWSCluster.Spec.Partition +} diff --git a/pkg/cloud/scope/ec2.go b/pkg/cloud/scope/ec2.go index 6ede8078e2..7d19fe4c8b 100644 --- a/pkg/cloud/scope/ec2.go +++ b/pkg/cloud/scope/ec2.go @@ -19,12 +19,19 @@ package scope import ( infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud" + "sigs.k8s.io/controller-runtime/pkg/client" ) // EC2Scope is the interface for the scope to be used with the ec2 service. type EC2Scope interface { cloud.ClusterScoper + // RemoteClient returns the Kubernetes Client for connecting to the workload cluster. + RemoteClient() (client.Client, error) + + // ManagementClient returns the Kubernetes Client for the management cluster. + ManagementClient() client.Client + // VPC returns the cluster VPC. VPC() *infrav1.VPCSpec @@ -40,6 +47,15 @@ type EC2Scope interface { // Bastion returns the bastion details for the cluster. Bastion() *infrav1.Bastion + // Bucket returns the s3 bucket details for the cluster. + Bucket() *infrav1.S3Bucket + + // AssociateOIDCProvider returns if the cluster should have an OIDC Provider Associated. + AssociateOIDCProvider() bool + + // OIDCProviderStatus returns the status of the OIDC provider associated to the cluster for IRSA + OIDCProviderStatus() *infrav1.OIDCProviderStatus + // SetBastionInstance sets the bastion instance in the status of the cluster. SetBastionInstance(instance *infrav1.Instance) diff --git a/pkg/cloud/scope/fargate.go b/pkg/cloud/scope/fargate.go index 00052905c5..09a9254c10 100644 --- a/pkg/cloud/scope/fargate.go +++ b/pkg/cloud/scope/fargate.go @@ -18,6 +18,7 @@ package scope import ( "context" + "sigs.k8s.io/cluster-api-provider-aws/util/system" awsclient "github.com/aws/aws-sdk-go/aws/client" "github.com/go-logr/logr" @@ -157,6 +158,14 @@ func (s *FargateProfileScope) SubnetIDs() []string { return s.FargateProfile.Spec.SubnetIDs } +// Partition returns the machine pool subnet IDs. +func (s *FargateProfileScope) Partition() string { + if s.ControlPlane.Spec.Partition == "" { + s.ControlPlane.Spec.Partition = system.GetPartitionFromRegion(s.ControlPlane.Spec.Region) + } + return s.ControlPlane.Spec.Partition +} + // IAMReadyFalse marks the ready condition false using warning if error isn't // empty. func (s *FargateProfileScope) IAMReadyFalse(reason string, err string) error { diff --git a/pkg/cloud/scope/iam.go b/pkg/cloud/scope/iam.go new file mode 100644 index 0000000000..668ed58e40 --- /dev/null +++ b/pkg/cloud/scope/iam.go @@ -0,0 +1,8 @@ +package scope + +import "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud" + +// IAMScope is the interface for the scope to be used with the IAM service. +type IAMScope interface { + cloud.ClusterScoper +} diff --git a/pkg/cloud/scope/launchtemplate.go b/pkg/cloud/scope/launchtemplate.go new file mode 100644 index 0000000000..180878855b --- /dev/null +++ b/pkg/cloud/scope/launchtemplate.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scope + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" + expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud" + expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" +) + +const ( + // AWSManagedControlPlaneKind is the Kind of AWSManagedControlPlane. + AWSManagedControlPlaneKind = "AWSManagedControlPlane" +) + +// LaunchTemplateScope defines a scope defined around a launch template. +type LaunchTemplateScope interface { + GetMachinePool() *expclusterv1.MachinePool + GetLaunchTemplate() *expinfrav1.AWSLaunchTemplate + LaunchTemplateName() string + GetLaunchTemplateIDStatus() string + SetLaunchTemplateIDStatus(id string) + GetLaunchTemplateLatestVersionStatus() string + SetLaunchTemplateLatestVersionStatus(version string) + GetRawBootstrapData() ([]byte, error) + + IsEKSManaged() bool + AdditionalTags() infrav1.Tags + + GetObjectMeta() *metav1.ObjectMeta + GetSetter() conditions.Setter + PatchObject() error + GetEC2Scope() EC2Scope + + client.Client + cloud.Logger +} + +type ResourceServiceToUpdate struct { + ResourceID *string + ResourceService ResourceService +} + +type ResourceService interface { + UpdateResourceTags(resourceID *string, create, remove map[string]string) error +} diff --git a/pkg/cloud/scope/machine.go b/pkg/cloud/scope/machine.go index 189e3d7f83..17931f53e6 100644 --- a/pkg/cloud/scope/machine.go +++ b/pkg/cloud/scope/machine.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "path" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -259,6 +260,11 @@ func (m *MachineScope) GetBootstrapData() (string, error) { return base64.StdEncoding.EncodeToString(value), nil } +// GetBootstrapDataKey returns the bootstrap data key to use in the s3 bucket when storing ignition bootstrap data. +func (m *MachineScope) GetBootstrapDataKey() string { + return path.Join(m.Role(), m.Name()) +} + // GetRawBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName. func (m *MachineScope) GetRawBootstrapData() ([]byte, error) { data, _, err := m.GetRawBootstrapDataWithFormat() @@ -363,7 +369,7 @@ func (m *MachineScope) AWSMachineIsDeleted() bool { // IsEKSManaged checks if the machine is EKS managed. func (m *MachineScope) IsEKSManaged() bool { - return m.InfraCluster.InfraCluster().GetObjectKind().GroupVersionKind().Kind == "AWSManagedControlPlane" + return m.InfraCluster.InfraCluster().GetObjectKind().GroupVersionKind().Kind == AWSManagedControlPlaneKind } // IsExternallyManaged checks if the machine is externally managed. diff --git a/pkg/cloud/scope/machinepool.go b/pkg/cloud/scope/machinepool.go index 16b35db937..d71f4fc480 100644 --- a/pkg/cloud/scope/machinepool.go +++ b/pkg/cloud/scope/machinepool.go @@ -19,6 +19,9 @@ package scope import ( "context" "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/util/conditions" "strings" "github.com/go-logr/logr" @@ -42,7 +45,7 @@ import ( // MachinePoolScope defines a scope defined around a machine and its cluster. type MachinePoolScope struct { logr.Logger - client client.Client + client.Client patchHelper *patch.Helper Cluster *clusterv1.Cluster @@ -53,7 +56,7 @@ type MachinePoolScope struct { // MachinePoolScopeParams defines a scope defined around a machine and its cluster. type MachinePoolScopeParams struct { - Client client.Client + client.Client Logger *logr.Logger Cluster *clusterv1.Cluster @@ -101,7 +104,7 @@ func NewMachinePoolScope(params MachinePoolScopeParams) (*MachinePoolScope, erro return &MachinePoolScope{ Logger: *params.Logger, - client: params.Client, + Client: params.Client, patchHelper: helper, Cluster: params.Cluster, @@ -141,7 +144,7 @@ func (m *MachinePoolScope) getBootstrapData() ([]byte, string, error) { secret := &corev1.Secret{} key := types.NamespacedName{Namespace: m.Namespace(), Name: *m.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName} - if err := m.client.Get(context.TODO(), key, secret); err != nil { + if err := m.Client.Get(context.TODO(), key, secret); err != nil { return nil, "", errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSMachine %s/%s", m.Namespace(), m.Name()) } @@ -220,14 +223,42 @@ func (m *MachinePoolScope) SetASGStatus(v expinfrav1.ASGStatus) { m.AWSMachinePool.Status.ASGStatus = &v } +func (m *MachinePoolScope) GetObjectMeta() *metav1.ObjectMeta { + return &m.AWSMachinePool.ObjectMeta +} + +func (m *MachinePoolScope) GetSetter() conditions.Setter { + return m.AWSMachinePool +} + +func (m *MachinePoolScope) GetEC2Scope() EC2Scope { + return m.InfraCluster +} + +func (m *MachinePoolScope) GetLaunchTemplateIDStatus() string { + return m.AWSMachinePool.Status.LaunchTemplateID +} + // SetLaunchTemplateIDStatus sets the AWSMachinePool LaunchTemplateID status. func (m *MachinePoolScope) SetLaunchTemplateIDStatus(id string) { m.AWSMachinePool.Status.LaunchTemplateID = id } +func (m *MachinePoolScope) GetLaunchTemplateLatestVersionStatus() string { + if m.AWSMachinePool.Status.LaunchTemplateVersion != nil { + return *m.AWSMachinePool.Status.LaunchTemplateVersion + } else { + return "" + } +} + +func (m *MachinePoolScope) SetLaunchTemplateLatestVersionStatus(version string) { + m.AWSMachinePool.Status.LaunchTemplateVersion = &version +} + // IsEKSManaged checks if the AWSMachinePool is EKS managed. func (m *MachinePoolScope) IsEKSManaged() bool { - return m.InfraCluster.InfraCluster().GetObjectKind().GroupVersionKind().Kind == "AWSManagedControlPlane" + return m.InfraCluster.InfraCluster().GetObjectKind().GroupVersionKind().Kind == AWSManagedControlPlaneKind } // SubnetIDs returns the machine pool subnet IDs. @@ -291,7 +322,7 @@ func (m *MachinePoolScope) getNodeStatusByProviderID(ctx context.Context, provid nodeStatusMap[id] = &NodeStatus{} } - workloadClient, err := remote.NewClusterClient(ctx, "", m.client, util.ObjectKey(m.Cluster)) + workloadClient, err := remote.NewClusterClient(ctx, "", m.Client, util.ObjectKey(m.Cluster)) if err != nil { return nil, err } @@ -327,3 +358,19 @@ func nodeIsReady(node corev1.Node) bool { } return false } + +func (m *MachinePoolScope) GetLaunchTemplate() *expinfrav1.AWSLaunchTemplate { + return &m.AWSMachinePool.Spec.AWSLaunchTemplate +} + +func (m *MachinePoolScope) GetMachinePool() *expclusterv1.MachinePool { + return m.MachinePool +} + +func (m *MachinePoolScope) LaunchTemplateName() string { + return m.Name() +} + +func (m *MachinePoolScope) GetRuntimeObject() runtime.Object { + return m.AWSMachinePool +} diff --git a/pkg/cloud/scope/managedcontrolplane.go b/pkg/cloud/scope/managedcontrolplane.go index 28b5b7392b..ee2d2a4ef9 100644 --- a/pkg/cloud/scope/managedcontrolplane.go +++ b/pkg/cloud/scope/managedcontrolplane.go @@ -19,16 +19,17 @@ package scope import ( "context" "fmt" - "time" - amazoncni "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" awsclient "github.com/aws/aws-sdk-go/aws/client" "github.com/go-logr/logr" "github.com/pkg/errors" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2/klogr" + "sigs.k8s.io/cluster-api-provider-aws/util/system" "sigs.k8s.io/controller-runtime/pkg/client" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -48,6 +49,8 @@ func init() { _ = amazoncni.AddToScheme(scheme) _ = appsv1.AddToScheme(scheme) _ = corev1.AddToScheme(scheme) + _ = rbacv1.AddToScheme(scheme) + _ = admissionregistrationv1.AddToScheme(scheme) } // ManagedControlPlaneScopeParams defines the input parameters used to create a new Scope. @@ -135,11 +138,16 @@ func (s *ManagedControlPlaneScope) RemoteClient() (client.Client, error) { if err != nil { return nil, fmt.Errorf("getting remote rest config for %s/%s: %w", s.Namespace(), s.Name(), err) } - restConfig.Timeout = 1 * time.Minute + restConfig.Timeout = DefaultKubeClientTimeout return client.New(restConfig, client.Options{Scheme: scheme}) } +// ManagementClient returns the Kubernetes Client for the management cluster. +func (s *ManagedControlPlaneScope) ManagementClient() client.Client { + return s.Client +} + // Network returns the control plane network object. func (s *ManagedControlPlaneScope) Network() *infrav1.NetworkStatus { return &s.ControlPlane.Status.Network @@ -236,6 +244,7 @@ func (s *ManagedControlPlaneScope) PatchObject() error { infrav1.NatGatewaysReadyCondition, infrav1.RouteTablesReadyCondition, infrav1.BastionHostReadyCondition, + infrav1.EgressOnlyInternetGatewayReadyCondition, ekscontrolplanev1.EKSControlPlaneCreatingCondition, ekscontrolplanev1.EKSControlPlaneReadyCondition, ekscontrolplanev1.EKSControlPlaneUpdatingCondition, @@ -290,6 +299,16 @@ func (s *ManagedControlPlaneScope) Bastion() *infrav1.Bastion { return &s.ControlPlane.Spec.Bastion } +// Bucket returns the s3 bucket details. +func (s *ManagedControlPlaneScope) Bucket() *infrav1.S3Bucket { + return nil // no s3 bucket for managed clusters +} + +// AssociateOIDCProvider returns if the cluster should have an OIDC Provider Associated. +func (s *ManagedControlPlaneScope) AssociateOIDCProvider() bool { + return s.ControlPlane.Spec.AssociateOIDCProvider +} + // SetBastionInstance sets the bastion instance in the status of the cluster. func (s *ManagedControlPlaneScope) SetBastionInstance(instance *infrav1.Instance) { s.ControlPlane.Status.Bastion = instance @@ -372,10 +391,19 @@ func (s *ManagedControlPlaneScope) DisableVPCCNI() bool { return s.ControlPlane.Spec.DisableVPCCNI } +// VpcCni returns a list of environment variables to apply to the `aws-node` DaemonSet. +func (s *ManagedControlPlaneScope) VpcCni() ekscontrolplanev1.VpcCni { + return s.ControlPlane.Spec.VpcCni +} + func (s *ManagedControlPlaneScope) OIDCIdentityProviderConfig() *ekscontrolplanev1.OIDCIdentityProviderConfig { return s.ControlPlane.Spec.OIDCIdentityProviderConfig } +func (s *ManagedControlPlaneScope) OIDCProviderStatus() *infrav1.OIDCProviderStatus { + return &s.ControlPlane.Status.OIDCProvider +} + // ServiceCidrs returns the CIDR blocks used for services. func (s *ManagedControlPlaneScope) ServiceCidrs() *clusterv1.NetworkRanges { if s.Cluster.Spec.ClusterNetwork != nil { @@ -388,3 +416,11 @@ func (s *ManagedControlPlaneScope) ServiceCidrs() *clusterv1.NetworkRanges { return nil } + +// Partition returns the cluster partition. +func (s *ManagedControlPlaneScope) Partition() string { + if s.ControlPlane.Spec.Partition == "" { + s.ControlPlane.Spec.Partition = system.GetPartitionFromRegion(s.Region()) + } + return s.ControlPlane.Spec.Partition +} diff --git a/pkg/cloud/scope/managednodegroup.go b/pkg/cloud/scope/managednodegroup.go index ca91a7f48f..a867e4e7b4 100644 --- a/pkg/cloud/scope/managednodegroup.go +++ b/pkg/cloud/scope/managednodegroup.go @@ -19,6 +19,11 @@ package scope import ( "context" "fmt" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api-provider-aws/util/system" awsclient "github.com/aws/aws-sdk-go/aws/client" "github.com/go-logr/logr" @@ -51,6 +56,8 @@ type ManagedMachinePoolScopeParams struct { EnableIAM bool AllowAdditionalRoles bool + + InfraCluster EC2Scope } // NewManagedMachinePoolScope creates a new Scope from the supplied parameters. @@ -94,6 +101,7 @@ func NewManagedMachinePoolScope(params ManagedMachinePoolScopeParams) (*ManagedM ControlPlane: params.ControlPlane, ManagedMachinePool: params.ManagedMachinePool, MachinePool: params.MachinePool, + EC2Scope: params.InfraCluster, patchHelper: helper, session: session, serviceLimiters: serviceLimiters, @@ -106,13 +114,14 @@ func NewManagedMachinePoolScope(params ManagedMachinePoolScopeParams) (*ManagedM // ManagedMachinePoolScope defines the basic context for an actuator to operate upon. type ManagedMachinePoolScope struct { logr.Logger - Client client.Client + client.Client patchHelper *patch.Helper Cluster *clusterv1.Cluster ControlPlane *ekscontrolplanev1.AWSManagedControlPlane ManagedMachinePool *expinfrav1.AWSManagedMachinePool MachinePool *expclusterv1.MachinePool + EC2Scope EC2Scope session awsclient.ConfigProvider serviceLimiters throttle.ServiceLimiters @@ -150,6 +159,11 @@ func (s *ManagedMachinePoolScope) AllowAdditionalRoles() bool { return s.allowAdditionalRoles } +// Partition returns the machine pool subnet IDs. +func (s *ManagedMachinePoolScope) Partition() string { + return system.GetPartitionFromRegion(s.ControlPlane.Spec.Region) +} + // IdentityRef returns the cluster identityRef. func (s *ManagedMachinePoolScope) IdentityRef() *infrav1.AWSIdentityReference { return s.ControlPlane.Spec.IdentityRef @@ -158,11 +172,14 @@ func (s *ManagedMachinePoolScope) IdentityRef() *infrav1.AWSIdentityReference { // AdditionalTags returns AdditionalTags from the scope's ManagedMachinePool // The returned value will never be nil. func (s *ManagedMachinePoolScope) AdditionalTags() infrav1.Tags { - if s.ManagedMachinePool.Spec.AdditionalTags == nil { - s.ManagedMachinePool.Spec.AdditionalTags = infrav1.Tags{} - } + tags := make(infrav1.Tags) + + // Start with the cluster-wide tags... + tags.Merge(s.EC2Scope.AdditionalTags()) + // ... and merge in the Machine's + tags.Merge(s.ManagedMachinePool.Spec.AdditionalTags) - return s.ManagedMachinePool.Spec.AdditionalTags.DeepCopy() + return tags } // RoleName returns the node group role name. @@ -281,3 +298,87 @@ func (s *ManagedMachinePoolScope) KubernetesClusterName() string { func (s *ManagedMachinePoolScope) NodegroupName() string { return s.ManagedMachinePool.Spec.EKSNodegroupName } + +func (s *ManagedMachinePoolScope) Name() string { + return s.ManagedMachinePool.Name +} + +func (s *ManagedMachinePoolScope) Namespace() string { + return s.ManagedMachinePool.Namespace +} + +func (s *ManagedMachinePoolScope) GetRawBootstrapData() ([]byte, error) { + if s.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { + return nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil") + } + + secret := &corev1.Secret{} + key := types.NamespacedName{Namespace: s.Namespace(), Name: *s.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName} + + if err := s.Client.Get(context.TODO(), key, secret); err != nil { + return nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSManagedMachinePool %s/%s", s.Namespace(), s.Name()) + } + + value, ok := secret.Data["value"] + if !ok { + return nil, errors.New("error retrieving bootstrap data: secret value key is missing") + } + + return value, nil +} + +func (s *ManagedMachinePoolScope) GetObjectMeta() *metav1.ObjectMeta { + return &s.ManagedMachinePool.ObjectMeta +} + +func (s *ManagedMachinePoolScope) GetSetter() conditions.Setter { + return s.ManagedMachinePool +} + +func (s *ManagedMachinePoolScope) GetEC2Scope() EC2Scope { + return s.EC2Scope +} + +func (s *ManagedMachinePoolScope) IsEKSManaged() bool { + return true +} + +func (s *ManagedMachinePoolScope) GetLaunchTemplateIDStatus() string { + if s.ManagedMachinePool.Status.LaunchTemplateID != nil { + return *s.ManagedMachinePool.Status.LaunchTemplateID + } else { + return "" + } +} + +func (s *ManagedMachinePoolScope) SetLaunchTemplateIDStatus(id string) { + s.ManagedMachinePool.Status.LaunchTemplateID = &id +} + +func (s *ManagedMachinePoolScope) GetLaunchTemplateLatestVersionStatus() string { + if s.ManagedMachinePool.Status.LaunchTemplateVersion != nil { + return *s.ManagedMachinePool.Status.LaunchTemplateVersion + } else { + return "" + } +} + +func (s *ManagedMachinePoolScope) SetLaunchTemplateLatestVersionStatus(version string) { + s.ManagedMachinePool.Status.LaunchTemplateVersion = &version +} + +func (s *ManagedMachinePoolScope) GetLaunchTemplate() *expinfrav1.AWSLaunchTemplate { + return s.ManagedMachinePool.Spec.AWSLaunchTemplate +} + +func (s *ManagedMachinePoolScope) GetMachinePool() *expclusterv1.MachinePool { + return s.MachinePool +} + +func (s *ManagedMachinePoolScope) LaunchTemplateName() string { + return fmt.Sprintf("%s-%s", s.ControlPlane.Name, s.ManagedMachinePool.Name) +} + +func (s *ManagedMachinePoolScope) GetRuntimeObject() runtime.Object { + return s.ManagedMachinePool +} diff --git a/pkg/cloud/scope/shared.go b/pkg/cloud/scope/shared.go index e9dfb68f12..a731f66135 100644 --- a/pkg/cloud/scope/shared.go +++ b/pkg/cloud/scope/shared.go @@ -18,6 +18,7 @@ package scope import ( "fmt" + "time" "github.com/go-logr/logr" "github.com/pkg/errors" @@ -33,6 +34,8 @@ var ( ErrLoggerRequired = errors.New("logger is required") // ErrNotPlaced is an error if there is no placement determined. ErrNotPlaced = errors.New("placement not determined") + + DefaultKubeClientTimeout = 1 * time.Minute ) type placementInput struct { diff --git a/pkg/cloud/services/autoscaling/autoscalinggroup.go b/pkg/cloud/services/autoscaling/autoscalinggroup.go index 7b917ed12f..033c7327da 100644 --- a/pkg/cloud/services/autoscaling/autoscalinggroup.go +++ b/pkg/cloud/services/autoscaling/autoscalinggroup.go @@ -83,8 +83,9 @@ func (s *Service) SDKToAutoScalingGroup(v *autoscaling.Group) (*expinfrav1.AutoS if len(v.Instances) > 0 { for _, autoscalingInstance := range v.Instances { tmp := &infrav1.Instance{ - ID: aws.StringValue(autoscalingInstance.InstanceId), - State: infrav1.InstanceState(*autoscalingInstance.LifecycleState), + ID: aws.StringValue(autoscalingInstance.InstanceId), + State: infrav1.InstanceState(*autoscalingInstance.LifecycleState), + AvailabilityZone: *autoscalingInstance.AvailabilityZone, } i.Instances = append(i.Instances, *tmp) } diff --git a/pkg/cloud/services/autoscaling/autoscalinggroup_test.go b/pkg/cloud/services/autoscaling/autoscalinggroup_test.go index d4d974a595..2010b7382e 100644 --- a/pkg/cloud/services/autoscaling/autoscalinggroup_test.go +++ b/pkg/cloud/services/autoscaling/autoscalinggroup_test.go @@ -218,8 +218,9 @@ func TestService_SDKToAutoScalingGroup(t *testing.T) { }, Instances: []*autoscaling.Instance{ { - InstanceId: aws.String("instanceId"), - LifecycleState: aws.String("lifecycleState"), + InstanceId: aws.String("instanceId"), + LifecycleState: aws.String("lifecycleState"), + AvailabilityZone: aws.String("us-east-1a"), }, }, }, @@ -249,8 +250,9 @@ func TestService_SDKToAutoScalingGroup(t *testing.T) { }, Instances: []infrav1.Instance{ { - ID: "instanceId", - State: "lifecycleState", + ID: "instanceId", + State: "lifecycleState", + AvailabilityZone: "us-east-1a", }, }, }, diff --git a/pkg/cloud/services/awsnode/cni.go b/pkg/cloud/services/awsnode/cni.go index 437ca1eb11..0239fecfb6 100644 --- a/pkg/cloud/services/awsnode/cni.go +++ b/pkg/cloud/services/awsnode/cni.go @@ -23,15 +23,17 @@ import ( amazoncni "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/api/konfig" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors" - "sigs.k8s.io/cluster-api-provider-aws/pkg/record" ) const ( @@ -53,9 +55,6 @@ func (s *Service) ReconcileCNI(ctx context.Context) error { if err := s.deleteCNI(ctx, remoteClient); err != nil { return fmt.Errorf("disabling aws vpc cni: %w", err) } - } - - if s.scope.SecondaryCidrBlock() == nil { return nil } @@ -67,6 +66,29 @@ func (s *Service) ReconcileCNI(ctx context.Context) error { return ErrCNIMissing } + var needsUpdate bool + if len(s.scope.VpcCni().Env) > 0 { + s.scope.Info("updating aws-node daemonset environment variables", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + if container.Name == "aws-node" { + container.Env, needsUpdate = s.applyUserProvidedEnvironmentProperties(container.Env) + + } + } + } + + if s.scope.SecondaryCidrBlock() == nil { + if needsUpdate { + s.scope.Info("adding environment properties to vpc-cni", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + if err = remoteClient.Update(ctx, &ds, &client.UpdateOptions{}); err != nil { + return err + } + } + return nil + } + sgs, err := s.getSecurityGroups() if err != nil { return err @@ -141,20 +163,6 @@ func (s *Service) ReconcileCNI(ctx context.Context) error { } s.scope.Info("updating containers", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) - for _, container := range ds.Spec.Template.Spec.Containers { - if container.Name == "aws-node" { - container.Env = append(s.filterEnv(container.Env), - corev1.EnvVar{ - Name: "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG", - Value: "true", - }, - corev1.EnvVar{ - Name: "ENI_CONFIG_LABEL_DEF", - Value: "failure-domain.beta.kubernetes.io/zone", - }, - ) - } - } return remoteClient.Update(ctx, &ds, &client.UpdateOptions{}) } @@ -175,39 +183,109 @@ func (s *Service) getSecurityGroups() ([]string, error) { return sgs, nil } -func (s *Service) filterEnv(env []corev1.EnvVar) []corev1.EnvVar { - var i int - for _, e := range env { - if e.Name == "ENI_CONFIG_LABEL_DEF" || e.Name == "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG" { - continue +// applyUserProvidedEnvironmentProperties takes a container environment and applies user provided values to it. +func (s *Service) applyUserProvidedEnvironmentProperties(containerEnv []corev1.EnvVar) ([]corev1.EnvVar, bool) { + var ( + envVars = make(map[string]corev1.EnvVar) + needsUpdate = false + ) + for _, e := range s.scope.VpcCni().Env { + envVars[e.Name] = e + } + // Handle the case where we overwrite an existing value if it's not already the desired value. + // This will prevent continuously updating the DaemonSet even though there are no changes. + for i, e := range containerEnv { + if v, ok := envVars[e.Name]; ok { + // Take care of comparing secret ref with Stringer. + if containerEnv[i].String() != v.String() { + needsUpdate = true + containerEnv[i] = v + } + delete(envVars, e.Name) } - env[i] = e - i++ } - return env[:i] + // Handle case when there are values that aren't in the list of environment properties + // of aws-node. + for _, v := range envVars { + needsUpdate = true + containerEnv = append(containerEnv, v) + } + return containerEnv, needsUpdate } func (s *Service) deleteCNI(ctx context.Context, remoteClient client.Client) error { - s.scope.Info("Ensuring aws-node DaemonSet in cluster is deleted", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + // EKS has a tendency to pre-install the vpc-cni automagically even if you don't specify it as an addon + // and looks like a kubectl apply from a script of a manifest that looks like this + // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/config/master/aws-k8s-cni.yaml + // and removing these pieces will enable someone to install and alternative CNI. There is also another use + // case where someone would want to remove the vpc-cni and reinstall it via the helm chart located here + // https://github.com/aws/amazon-vpc-cni-k8s/tree/master/charts/aws-vpc-cni meaning we need to account for + // managed-by: Helm label, or we will delete the helm chart resources every reconcile loop. EKS does make + // a CRD for eniconfigs but the default env var on the vpc-cni pod is ENABLE_POD_ENI=false. We will make an + // assumption no CRs are ever created and leave the CRD to reduce complexity of this operation. - ds := &appsv1.DaemonSet{} - if err := remoteClient.Get(ctx, types.NamespacedName{Namespace: awsNodeNamespace, Name: awsNodeName}, ds); err != nil { - if apierrors.IsNotFound(err) { - s.scope.V(2).Info("The aws-node DaemonSet is not found, not action") - return nil - } - return fmt.Errorf("getting aws-node daemonset: %w", err) + s.scope.Info("Ensuring all resources for AWS VPC CNI in cluster are deleted", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + + s.scope.Info("Trying to delete AWS VPC CNI DaemonSet", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + if err := s.deleteResource(ctx, remoteClient, types.NamespacedName{ + Namespace: awsNodeNamespace, + Name: awsNodeName, + }, &appsv1.DaemonSet{}); err != nil { + return err + } + + s.scope.Info("Trying to delete AWS VPC CNI ServiceAccount", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + if err := s.deleteResource(ctx, remoteClient, types.NamespacedName{ + Namespace: awsNodeNamespace, + Name: awsNodeName, + }, &corev1.ServiceAccount{}); err != nil { + return err + } + + s.scope.Info("Trying to delete AWS VPC CNI ClusterRoleBinding", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + if err := s.deleteResource(ctx, remoteClient, types.NamespacedName{ + Namespace: string(meta.RESTScopeNameRoot), + Name: awsNodeName, + }, &rbacv1.ClusterRoleBinding{}); err != nil { + return err } - s.scope.V(2).Info("The aws-node DaemonSet found, deleting") - if err := remoteClient.Delete(ctx, ds, &client.DeleteOptions{}); err != nil { - if apierrors.IsNotFound(err) { - s.scope.V(2).Info("The aws-node DaemonSet is not found, not deleted") - return nil + s.scope.Info("Trying to delete AWS VPC CNI ClusterRole", "cluster-name", s.scope.Name(), "cluster-namespace", s.scope.Namespace()) + if err := s.deleteResource(ctx, remoteClient, types.NamespacedName{ + Namespace: string(meta.RESTScopeNameRoot), + Name: awsNodeName, + }, &rbacv1.ClusterRole{}); err != nil { + return err + } + + // Disable in palette to prevent repetitive message + //record.Eventf(s.scope.InfraCluster(), "DeletedVPCCNI", "The AWS VPC CNI has been removed from the cluster. Ensure you enable a CNI via another mechanism") + + return nil +} + +func (s *Service) deleteResource(ctx context.Context, remoteClient client.Client, key client.ObjectKey, obj client.Object) error { + if err := remoteClient.Get(ctx, key, obj); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting resource %s: %w", key, err) + } + s.scope.V(2).Info(fmt.Sprintf("resource %s was not found, no action", key)) + } else { + // resource found, delete if no label or not managed by helm + if val, ok := obj.GetLabels()[konfig.ManagedbyLabelKey]; !ok || val != "Helm" { + if err := remoteClient.Delete(ctx, obj, &client.DeleteOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting %s: %w", key, err) + } + s.scope.V(2).Info(fmt.Sprintf( + "resource %s was not found, not deleted", key)) + } else { + s.scope.V(2).Info(fmt.Sprintf("resource %s was deleted", key)) + } + } else { + s.scope.V(2).Info(fmt.Sprintf("resource %s is managed by helm, not deleted", key)) } - return fmt.Errorf("deleting aws-node DaemonSet: %w", err) } - record.Eventf(s.scope.InfraCluster(), "DeletedVPCCNI", "The AWS VPC CNI has been removed from the cluster. Ensure you enable a CNI via another mechanism") return nil } diff --git a/pkg/cloud/services/awsnode/cni_test.go b/pkg/cloud/services/awsnode/cni_test.go new file mode 100644 index 0000000000..e45f9f69d0 --- /dev/null +++ b/pkg/cloud/services/awsnode/cni_test.go @@ -0,0 +1,308 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsnode + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" +) + +func TestReconcileCniVpcCniValues(t *testing.T) { + tests := []struct { + name string + cniValues ekscontrolplanev1.VpcCni + daemonSet *v1.DaemonSet + consistsOf []corev1.EnvVar + }{ + { + name: "users can set environment values", + cniValues: ekscontrolplanev1.VpcCni{ + Env: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE1", + }, + }, + }, + daemonSet: &v1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: awsNodeName, + Namespace: awsNodeNamespace, + }, + Spec: v1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: awsNodeName, + Env: []corev1.EnvVar{}, + }, + }, + }, + }, + }, + }, + consistsOf: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE1", + }, + }, + }, + { + name: "users can set environment values without duplications", + cniValues: ekscontrolplanev1.VpcCni{ + Env: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE1", + }, + { + Name: "NAME1", + Value: "VALUE2", + }, + }, + }, + daemonSet: &v1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: awsNodeName, + Namespace: awsNodeNamespace, + }, + Spec: v1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: awsNodeName, + Env: []corev1.EnvVar{}, + }, + }, + }, + }, + }, + }, + consistsOf: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE2", + }, + }, + }, + { + name: "users can set environment values overwriting existing values", + cniValues: ekscontrolplanev1.VpcCni{ + Env: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE1", + }, + { + Name: "NAME2", + Value: "VALUE2", + }, + }, + }, + daemonSet: &v1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: awsNodeName, + Namespace: awsNodeNamespace, + }, + Spec: v1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: awsNodeName, + Env: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "OVERWRITE", + }, + { + Name: "NAME3", + Value: "VALUE3", + }, + }, + }, + }, + }, + }, + }, + }, + consistsOf: []corev1.EnvVar{ + { + Name: "NAME1", + Value: "VALUE1", + }, + { + Name: "NAME2", + Value: "VALUE2", + }, + { + Name: "NAME3", + Value: "VALUE3", + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name+" without secondary cidr", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + g := NewWithT(t) + mockClient := &cachingClient{ + getValue: tc.daemonSet, + } + m := &mockScope{ + client: mockClient, + cni: tc.cniValues, + } + s := NewService(m) + + err := s.ReconcileCNI(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mockClient.updateChain).NotTo(BeEmpty()) + ds, ok := mockClient.updateChain[0].(*v1.DaemonSet) + g.Expect(ok).To(BeTrue()) + g.Expect(ds.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + g.Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(tc.consistsOf)) + }) + } + + for _, tc := range tests { + t.Run(tc.name+" with secondary cidr", func(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + g := NewWithT(t) + mockClient := &cachingClient{ + getValue: tc.daemonSet, + } + m := &mockScope{ + client: mockClient, + cni: tc.cniValues, + secondaryCidrBlock: aws.String("100.0.0.1/20"), + securityGroups: map[infrav1.SecurityGroupRole]infrav1.SecurityGroup{ + "node": { + ID: "sg-1234", + Name: "node", + }, + }, + } + s := NewService(m) + + err := s.ReconcileCNI(context.Background()) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mockClient.updateChain).NotTo(BeEmpty()) + ds, ok := mockClient.updateChain[0].(*v1.DaemonSet) + g.Expect(ok).To(BeTrue()) + g.Expect(ds.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + g.Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(tc.consistsOf)) + }) + } +} + +type cachingClient struct { + client.Client + getValue client.Object + updateChain []client.Object +} + +func (c *cachingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if _, ok := obj.(*v1.DaemonSet); ok { + daemonset, _ := obj.(*v1.DaemonSet) + *daemonset = *c.getValue.(*v1.DaemonSet) + } + return nil +} + +func (c *cachingClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + c.updateChain = append(c.updateChain, obj) + return nil +} + +func (c *cachingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return nil +} + +type mockScope struct { + scope.AWSNodeScope + client client.Client + cni ekscontrolplanev1.VpcCni + secondaryCidrBlock *string + securityGroups map[infrav1.SecurityGroupRole]infrav1.SecurityGroup + subnets infrav1.Subnets +} + +func (s *mockScope) RemoteClient() (client.Client, error) { + return s.client, nil +} + +func (s *mockScope) VpcCni() ekscontrolplanev1.VpcCni { + return s.cni +} + +func (s *mockScope) Info(msg string, keysAndValues ...interface{}) { + +} + +func (s *mockScope) Name() string { + return "mock-name" +} + +func (s *mockScope) Namespace() string { + return "mock-namespace" +} + +func (s *mockScope) DisableVPCCNI() bool { + return false +} + +func (s *mockScope) SecondaryCidrBlock() *string { + return s.secondaryCidrBlock +} + +func (s *mockScope) SecurityGroups() map[infrav1.SecurityGroupRole]infrav1.SecurityGroup { + return s.securityGroups +} + +func (s *mockScope) Subnets() infrav1.Subnets { + return s.subnets +} diff --git a/pkg/cloud/services/ec2/ami.go b/pkg/cloud/services/ec2/ami.go index c9e0621069..bc94d54ecd 100644 --- a/pkg/cloud/services/ec2/ami.go +++ b/pkg/cloud/services/ec2/ami.go @@ -19,6 +19,8 @@ package ec2 import ( "bytes" "fmt" + "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/bootstrap/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/util/system" "sort" "strings" "text/template" @@ -44,6 +46,8 @@ const ( // https://ubuntu.com/server/docs/cloud-images/amazon-ec2 ubuntuOwnerID = "099720109477" + ubuntuOwnerIDUsGov = "513442679011" + // Description regex for fetching Ubuntu AMIs for bastion host. ubuntuImageDescription = "Canonical??Ubuntu??20.04?LTS??amd64?focal?image*" @@ -195,13 +199,9 @@ func GetLatestImage(imgs []*ec2.Image) (*ec2.Image, error) { return imgs[len(imgs)-1], nil } -func (s *Service) defaultBastionAMILookup() (string, error) { +func (s *Service) defaultBastionAMILookup(region string) (string, error) { describeImageInput := &ec2.DescribeImagesInput{ Filters: []*ec2.Filter{ - { - Name: aws.String("owner-id"), - Values: []*string{aws.String(ubuntuOwnerID)}, - }, { Name: aws.String("architecture"), Values: []*string{aws.String("x86_64")}, @@ -220,6 +220,19 @@ func (s *Service) defaultBastionAMILookup() (string, error) { }, }, } + + ownerID := ubuntuOwnerID + partition := system.GetPartitionFromRegion(s.scope.Region()) + if strings.Contains(partition, v1beta1.PartitionNameUSGov) { + ownerID = ubuntuOwnerIDUsGov + } + + filter := &ec2.Filter{ + Name: aws.String("owner-id"), + Values: []*string{aws.String(ownerID)}, + } + describeImageInput.Filters = append(describeImageInput.Filters, filter) + out, err := s.EC2Client.DescribeImages(describeImageInput) if err != nil { return "", errors.Wrapf(err, "failed to describe images within region: %q", s.scope.Region()) diff --git a/pkg/cloud/services/ec2/bastion.go b/pkg/cloud/services/ec2/bastion.go index ea9610962d..12989e33f4 100644 --- a/pkg/cloud/services/ec2/bastion.go +++ b/pkg/cloud/services/ec2/bastion.go @@ -36,6 +36,7 @@ import ( const ( defaultSSHKeyName = "default" + defaultUsGovPartitionName = "us-gov" ) var ( @@ -76,7 +77,7 @@ func (s *Service) ReconcileBastion() error { return errors.Wrap(err, "failed to patch conditions") } } - defaultBastion, err := s.getDefaultBastion(s.scope.Bastion().InstanceType, s.scope.Bastion().AMI) + defaultBastion, err := s.getDefaultBastion(s.scope.Bastion().InstanceType, s.scope.Bastion().AMI, instance) if err != nil { record.Warnf(s.scope.InfraCluster(), "FailedFetchingBastion", "Failed to fetch default bastion instance: %v", err) return err @@ -166,7 +167,7 @@ func (s *Service) describeBastionInstance() (*infrav1.Instance, error) { return nil, awserrors.NewNotFound("bastion host not found") } -func (s *Service) getDefaultBastion(instanceType, ami string) (*infrav1.Instance, error) { +func (s *Service) getDefaultBastion(instanceType, ami string, instance *infrav1.Instance) (*infrav1.Instance, error) { name := fmt.Sprintf("%s-bastion", s.scope.Name()) userData, _ := userdata.NewBastion(&userdata.BastionInput{}) @@ -186,9 +187,10 @@ func (s *Service) getDefaultBastion(instanceType, ami string) (*infrav1.Instance } } + region := s.scope.Region() if ami == "" { var err error - ami, err = s.defaultBastionAMILookup() + ami, err = s.defaultBastionAMILookup(region) if err != nil { return nil, err } diff --git a/pkg/cloud/services/ec2/bastion_test.go b/pkg/cloud/services/ec2/bastion_test.go index 18f5577ae7..e37e12880f 100644 --- a/pkg/cloud/services/ec2/bastion_test.go +++ b/pkg/cloud/services/ec2/bastion_test.go @@ -307,10 +307,242 @@ func TestService_ReconcileBastion(t *testing.T) { m.DescribeInstances(gomock.Eq(describeInput)). Return(&ec2.DescribeInstancesOutput{}, nil).MinTimes(1) m.DescribeImages(gomock.Eq(&ec2.DescribeImagesInput{Filters: []*ec2.Filter{ + { + Name: aws.String("architecture"), + Values: aws.StringSlice([]string{"x86_64"}), + }, + { + Name: aws.String("state"), + Values: aws.StringSlice([]string{"available"}), + }, + { + Name: aws.String("virtualization-type"), + Values: aws.StringSlice([]string{"hvm"}), + }, + { + Name: aws.String("description"), + Values: aws.StringSlice([]string{ubuntuImageDescription}), + }, { Name: aws.String("owner-id"), Values: aws.StringSlice([]string{ubuntuOwnerID}), }, + }})).Return(&ec2.DescribeImagesOutput{Images: images{ + { + ImageId: aws.String("ubuntu-ami-id-latest"), + CreationDate: aws.String("2019-02-08T17:02:31.000Z"), + }, + { + ImageId: aws.String("ubuntu-ami-id-old"), + CreationDate: aws.String("2014-02-08T17:02:31.000Z"), + }, + }}, nil) + m.RunInstances(gomock.Any()). + Return(&ec2.Reservation{ + Instances: []*ec2.Instance{ + { + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + IamInstanceProfile: &ec2.IamInstanceProfile{ + Arn: aws.String("arn:aws:iam::123456789012:instance-profile/foo"), + }, + InstanceId: aws.String("id123"), + InstanceType: aws.String("t3.micro"), + SubnetId: aws.String("subnet-1"), + ImageId: aws.String("ubuntu-ami-id-latest"), + RootDeviceName: aws.String("device-1"), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + { + DeviceName: aws.String("device-1"), + Ebs: &ec2.EbsInstanceBlockDevice{ + VolumeId: aws.String("volume-1"), + }, + }, + }, + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("us-east-1"), + }, + }, + }, + }, nil) + m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil) + }, + bastionEnabled: true, + expectError: false, + bastionStatus: &infrav1.Instance{ + ID: "id123", + State: "running", + Type: "t3.micro", + SubnetID: "subnet-1", + ImageID: "ubuntu-ami-id-latest", + IAMProfile: "foo", + Addresses: []clusterv1.MachineAddress{}, + AvailabilityZone: "us-east-1", + VolumeIDs: []string{"volume-1"}, + }, + }, + } + + for _, tc := range tests { + managedValues := []bool{false, true} + for i := range managedValues { + managed := managedValues[i] + + t.Run(fmt.Sprintf("managed=%t %s", managed, tc.name), func(t *testing.T) { + g := NewWithT(t) + + mockControl := gomock.NewController(t) + defer mockControl.Finish() + + ec2Mock := mocks.NewMockEC2API(mockControl) + + scheme, err := setupScheme() + g.Expect(err).To(BeNil()) + + awsCluster := &infrav1.AWSCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: infrav1.AWSClusterSpec{ + NetworkSpec: infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpcID", + }, + Subnets: infrav1.Subnets{ + { + ID: "subnet-1", + }, + { + ID: "subnet-2", + IsPublic: true, + }, + }, + }, + Bastion: infrav1.Bastion{Enabled: tc.bastionEnabled}, + }, + } + + client := fake.NewClientBuilder().WithScheme(scheme).Build() + ctx := context.TODO() + client.Create(ctx, awsCluster) + + scope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: clusterName, + }, + }, + AWSCluster: awsCluster, + Client: client, + }) + g.Expect(err).To(BeNil()) + + if managed { + scope.AWSCluster.Spec.NetworkSpec.VPC.Tags = infrav1.Tags{ + infrav1.ClusterTagKey(clusterName): string(infrav1.ResourceLifecycleOwned), + } + } + + tc.expect(ec2Mock.EXPECT()) + s := NewService(scope) + s.EC2Client = ec2Mock + + err = s.ReconcileBastion() + if tc.expectError { + g.Expect(err).NotTo(BeNil()) + return + } + + g.Expect(err).To(BeNil()) + + g.Expect(scope.AWSCluster.Status.Bastion).To(BeEquivalentTo(tc.bastionStatus)) + }) + } + } +} + +func TestServiceReconcileBastionUSGOV(t *testing.T) { + clusterName := "cluster-us-gov" + + describeInput := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + filter.EC2.ProviderRole(infrav1.BastionRoleTagValue), + filter.EC2.Cluster(clusterName), + filter.EC2.InstanceStates( + ec2.InstanceStateNamePending, + ec2.InstanceStateNameRunning, + ec2.InstanceStateNameStopping, + ec2.InstanceStateNameStopped, + ), + }, + } + + foundOutput := &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + InstanceId: aws.String("id123"), + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("us-gov-east-1"), + }, + }, + }, + }, + }, + } + + tests := []struct { + name string + bastionEnabled bool + expect func(m *mocks.MockEC2APIMockRecorder) + expectError bool + bastionStatus *infrav1.Instance + }{ + { + name: "Should ignore reconciliation if instance not found", + expect: func(m *mocks.MockEC2APIMockRecorder) { + m. + DescribeInstances(gomock.Eq(describeInput)). + Return(&ec2.DescribeInstancesOutput{}, nil) + }, + expectError: false, + }, + { + name: "Should fail reconcile if describe instance fails", + expect: func(m *mocks.MockEC2APIMockRecorder) { + m. + DescribeInstances(gomock.Eq(describeInput)). + Return(nil, errors.New("some error")) + }, + expectError: true, + }, + { + name: "Should fail reconcile if terminate instance fails", + expect: func(m *mocks.MockEC2APIMockRecorder) { + m. + DescribeInstances(gomock.Eq(describeInput)). + Return(foundOutput, nil).MinTimes(1) + m. + TerminateInstances( + gomock.Eq(&ec2.TerminateInstancesInput{ + InstanceIds: aws.StringSlice([]string{"id123"}), + }), + ). + Return(nil, errors.New("some error")) + }, + expectError: true, + }, + { + name: "Should create bastion successfully", + expect: func(m *mocks.MockEC2APIMockRecorder) { + m.DescribeInstances(gomock.Eq(describeInput)). + Return(&ec2.DescribeInstancesOutput{}, nil).MinTimes(1) + m.DescribeImages(gomock.Eq(&ec2.DescribeImagesInput{Filters: []*ec2.Filter{ { Name: aws.String("architecture"), Values: aws.StringSlice([]string{"x86_64"}), @@ -327,6 +559,10 @@ func TestService_ReconcileBastion(t *testing.T) { Name: aws.String("description"), Values: aws.StringSlice([]string{ubuntuImageDescription}), }, + { + Name: aws.String("owner-id"), + Values: aws.StringSlice([]string{ubuntuOwnerIDUsGov}), + }, }})).Return(&ec2.DescribeImagesOutput{Images: images{ { ImageId: aws.String("ubuntu-ami-id-latest"), @@ -345,7 +581,7 @@ func TestService_ReconcileBastion(t *testing.T) { Name: aws.String(ec2.InstanceStateNameRunning), }, IamInstanceProfile: &ec2.IamInstanceProfile{ - Arn: aws.String("arn:aws:iam::123456789012:instance-profile/foo"), + Arn: aws.String("arn:aws-us-gov:iam::123456789012:instance-profile/foo"), }, InstanceId: aws.String("id123"), InstanceType: aws.String("t3.micro"), @@ -361,7 +597,7 @@ func TestService_ReconcileBastion(t *testing.T) { }, }, Placement: &ec2.Placement{ - AvailabilityZone: aws.String("us-east-1"), + AvailabilityZone: aws.String("us-gov-east-1"), }, }, }, @@ -379,7 +615,7 @@ func TestService_ReconcileBastion(t *testing.T) { ImageID: "ubuntu-ami-id-latest", IAMProfile: "foo", Addresses: []clusterv1.MachineAddress{}, - AvailabilityZone: "us-east-1", + AvailabilityZone: "us-gov-east-1", VolumeIDs: []string{"volume-1"}, }, }, @@ -419,6 +655,7 @@ func TestService_ReconcileBastion(t *testing.T) { }, }, Bastion: infrav1.Bastion{Enabled: tc.bastionEnabled}, + Region: "us-gov-east-1", }, } diff --git a/pkg/cloud/services/ec2/instances.go b/pkg/cloud/services/ec2/instances.go index 454ffe2be4..cce7697ef3 100644 --- a/pkg/cloud/services/ec2/instances.go +++ b/pkg/cloud/services/ec2/instances.go @@ -110,6 +110,8 @@ func (s *Service) InstanceIfExists(id *string) (*infrav1.Instance, error) { } // CreateInstance runs an ec2 instance. +// +//nolint:gocyclo // this function has multiple processes to perform func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte, userDataFormat string) (*infrav1.Instance, error) { s.scope.V(2).Info("Creating an instance for a machine") @@ -249,6 +251,27 @@ func (s *Service) CreateInstance(scope *scope.MachineScope, userData []byte, use } } + s.scope.V(2).Info("Adding tags on each network interface from resource", "resource-id", out.ID) + + // Fetching the network interfaces attached to the specific instanace + networkInterfaces, err := s.getInstanceENIs(out.ID) + if err != nil { + return nil, err + } + + s.scope.V(2).Info("Fetched the network interfaces") + + // Once all the network interfaces attached to the specific instanace are found, the similar tags of instance are created for network interfaces too + if len(networkInterfaces) > 0 { + s.scope.V(2).Info("Attempting to create tags from resource", "resource-id", out.ID) + for _, networkInterface := range networkInterfaces { + // Create/Update tags in AWS. + if err := s.UpdateResourceTags(networkInterface.NetworkInterfaceId, out.Tags, nil); err != nil { + return nil, errors.Wrapf(err, "failed to create tags for resource %q: ", *networkInterface.NetworkInterfaceId) + } + } + } + record.Eventf(scope.AWSMachine, "SuccessfulCreate", "Created new %s instance with id %q", scope.Role(), out.ID) return out, nil } @@ -409,7 +432,7 @@ func (s *Service) GetCoreSecurityGroups(scope *scope.MachineScope) ([]string, er // GetCoreNodeSecurityGroups looks up the security group IDs managed by this actuator // They are considered "core" to its proper functioning. -func (s *Service) GetCoreNodeSecurityGroups(scope *scope.MachinePoolScope) ([]string, error) { +func (s *Service) GetCoreNodeSecurityGroups(scope scope.LaunchTemplateScope) ([]string, error) { // These are common across both controlplane and node machines sgRoles := []infrav1.SecurityGroupRole{ infrav1.SecurityGroupNode, diff --git a/pkg/cloud/services/ec2/instances_test.go b/pkg/cloud/services/ec2/instances_test.go index cb74e17e1b..3b8e19d665 100644 --- a/pkg/cloud/services/ec2/instances_test.go +++ b/pkg/cloud/services/ec2/instances_test.go @@ -387,6 +387,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -494,6 +500,12 @@ func TestCreateInstance(t *testing.T) { m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -628,6 +640,12 @@ func TestCreateInstance(t *testing.T) { m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -758,6 +776,12 @@ func TestCreateInstance(t *testing.T) { m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -889,6 +913,12 @@ func TestCreateInstance(t *testing.T) { m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -996,6 +1026,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -1102,6 +1138,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -1283,6 +1325,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -1539,6 +1587,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -1738,6 +1792,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -1835,6 +1895,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2003,6 +2069,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2138,6 +2210,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2275,6 +2353,12 @@ func TestCreateInstance(t *testing.T) { }, nil) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2383,6 +2467,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2492,6 +2582,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2602,6 +2698,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2709,6 +2811,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2816,6 +2924,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { @@ -2923,6 +3037,12 @@ func TestCreateInstance(t *testing.T) { }) m.WaitUntilInstanceRunningWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil) + m. + DescribeNetworkInterfaces(gomock.Any()). + Return(&ec2.DescribeNetworkInterfacesOutput{ + NetworkInterfaces: []*ec2.NetworkInterface{}, + NextToken: nil, + }, nil) }, check: func(instance *infrav1.Instance, err error) { if err != nil { diff --git a/pkg/cloud/services/ec2/launchtemplate.go b/pkg/cloud/services/ec2/launchtemplate.go index 84eade6b9d..d15761bb9d 100644 --- a/pkg/cloud/services/ec2/launchtemplate.go +++ b/pkg/cloud/services/ec2/launchtemplate.go @@ -18,6 +18,7 @@ package ec2 import ( "encoding/base64" + "encoding/json" "fmt" "sort" "strconv" @@ -27,6 +28,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" @@ -34,8 +36,288 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/userdata" + "sigs.k8s.io/cluster-api-provider-aws/pkg/record" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" ) +const ( + // TagsLastAppliedAnnotation is the key for the AWSMachinePool object annotation + // which tracks the tags that the AWSMachinePool actuator is responsible + // for. These are the tags that have been handled by the + // AdditionalTags in the AWSMachinePool Provider Config. + // See https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + // for annotation formatting rules. + TagsLastAppliedAnnotation = "sigs.k8s.io/cluster-api-provider-aws-last-applied-tags" +) + +func (s *Service) ReconcileLaunchTemplate( + scope scope.LaunchTemplateScope, + canUpdateLaunchTemplate func() (bool, error), + runPostLaunchTemplateUpdateOperation func() error, +) error { + bootstrapData, err := scope.GetRawBootstrapData() + if err != nil { + record.Eventf(scope.GetMachinePool(), corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error()) + } + bootstrapDataHash := userdata.ComputeHash(bootstrapData) + + ec2svc := NewService(scope.GetEC2Scope()) + + scope.Info("checking for existing launch template") + launchTemplate, launchTemplateUserDataHash, err := ec2svc.GetLaunchTemplate(scope.LaunchTemplateName()) + if err != nil { + conditions.MarkUnknown(scope.GetSetter(), expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateNotFoundReason, err.Error()) + return err + } + + imageID, err := ec2svc.DiscoverLaunchTemplateAMI(scope) + if err != nil { + conditions.MarkFalse(scope.GetSetter(), expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateCreateFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return err + } + + if launchTemplate == nil { + scope.Info("no existing launch template found, creating") + launchTemplateID, err := ec2svc.CreateLaunchTemplate(scope, imageID, bootstrapData) + if err != nil { + conditions.MarkFalse(scope.GetSetter(), expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateCreateFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return err + } + + scope.SetLaunchTemplateIDStatus(launchTemplateID) + return scope.PatchObject() + } + + // LaunchTemplateID is set during LaunchTemplate creation, but for a scenario such as `clusterctl move`, status fields become blank. + // If launchTemplate already exists but LaunchTemplateID field in the status is empty, get the ID and update the status. + if scope.GetLaunchTemplateIDStatus() == "" { + launchTemplateID, err := ec2svc.GetLaunchTemplateID(scope.LaunchTemplateName()) + if err != nil { + conditions.MarkUnknown(scope.GetSetter(), expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateNotFoundReason, err.Error()) + return err + } + scope.SetLaunchTemplateIDStatus(launchTemplateID) + return scope.PatchObject() + } + + if scope.GetLaunchTemplateLatestVersionStatus() == "" { + launchTemplateVersion, err := ec2svc.GetLaunchTemplateLatestVersion(scope.GetLaunchTemplateIDStatus()) + if err != nil { + conditions.MarkUnknown(scope.GetSetter(), expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateNotFoundReason, err.Error()) + return err + } + scope.SetLaunchTemplateLatestVersionStatus(launchTemplateVersion) + return scope.PatchObject() + } + + annotation, err := MachinePoolAnnotationJSON(scope, TagsLastAppliedAnnotation) + if err != nil { + return err + } + + // Check if the instance tags were changed. If they were, create a new LaunchTemplate. + tagsChanged, _, _, _ := tagsChanged(annotation, scope.AdditionalTags()) // nolint:dogsled + + needsUpdate, err := ec2svc.LaunchTemplateNeedsUpdate(scope, scope.GetLaunchTemplate(), launchTemplate) + if err != nil { + return err + } + + if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID { + canUpdate, err := canUpdateLaunchTemplate() + if err != nil { + return err + } + if !canUpdate { + conditions.MarkFalse(scope.GetSetter(), expinfrav1.PreLaunchTemplateUpdateCheckCondition, expinfrav1.PreLaunchTemplateUpdateCheckFailedReason, clusterv1.ConditionSeverityWarning, "") + return errors.New("Cannot update the launch template, prerequisite not met") + } + } + + // Create a new launch template version if there's a difference in configuration, tags, + // userdata, OR we've discovered a new AMI ID. + if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID || launchTemplateUserDataHash != bootstrapDataHash { + scope.Info("creating new version for launch template", "existing", launchTemplate, "incoming", scope.GetLaunchTemplate()) + // There is a limit to the number of Launch Template Versions. + // We ensure that the number of versions does not grow without bound by following a simple rule: Before we create a new version, we delete one old version, if there is at least one old version that is not in use. + if err := ec2svc.PruneLaunchTemplateVersions(scope.GetLaunchTemplateIDStatus()); err != nil { + return err + } + if err := ec2svc.CreateLaunchTemplateVersion(scope.GetLaunchTemplateIDStatus(), scope, imageID, bootstrapData); err != nil { + return err + } + version, err := ec2svc.GetLaunchTemplateLatestVersion(scope.GetLaunchTemplateIDStatus()) + if err != nil { + return err + } + + scope.SetLaunchTemplateLatestVersionStatus(version) + return scope.PatchObject() + } + + if needsUpdate || tagsChanged || *imageID != *launchTemplate.AMI.ID { + if err := runPostLaunchTemplateUpdateOperation(); err != nil { + conditions.MarkFalse(scope.GetSetter(), expinfrav1.PostLaunchTemplateUpdateOperationCondition, expinfrav1.PostLaunchTemplateUpdateOperationFailedReason, clusterv1.ConditionSeverityError, err.Error()) + return err + } + conditions.MarkTrue(scope.GetSetter(), expinfrav1.PostLaunchTemplateUpdateOperationCondition) + } + + return nil +} + +func (s *Service) ReconcileTags(scope scope.LaunchTemplateScope, resourceServicesToUpdate []scope.ResourceServiceToUpdate) error { + additionalTags := scope.AdditionalTags() + + _, err := s.ensureTags(scope, resourceServicesToUpdate, additionalTags) + if err != nil { + return err + } + return nil +} + +func (s *Service) ensureTags(scope scope.LaunchTemplateScope, resourceServicesToUpdate []scope.ResourceServiceToUpdate, additionalTags map[string]string) (bool, error) { + annotation, err := MachinePoolAnnotationJSON(scope, TagsLastAppliedAnnotation) + if err != nil { + return false, err + } + + // Check if the instance tags were changed. If they were, update them. + // It would be possible here to only send new/updated tags, but for the + // moment we send everything, even if only a single tag was created or + // upated. + changed, created, deleted, newAnnotation := tagsChanged(annotation, additionalTags) + if changed { + for _, resourceServiceToUpdate := range resourceServicesToUpdate { + err := resourceServiceToUpdate.ResourceService.UpdateResourceTags(resourceServiceToUpdate.ResourceID, created, deleted) + if err != nil { + return false, err + } + } + + // We also need to update the annotation if anything changed. + err = UpdateMachinePoolAnnotationJSON(scope, TagsLastAppliedAnnotation, newAnnotation) + if err != nil { + return false, err + } + } + + return changed, nil +} + +func MachinePoolAnnotationJSON(lts scope.LaunchTemplateScope, annotation string) (map[string]interface{}, error) { + out := map[string]interface{}{} + + jsonAnnotation := machinePoolAnnotation(lts, annotation) + if len(jsonAnnotation) == 0 { + return out, nil + } + + err := json.Unmarshal([]byte(jsonAnnotation), &out) + if err != nil { + return out, err + } + + return out, nil +} + +func machinePoolAnnotation(lts scope.LaunchTemplateScope, annotation string) string { + return lts.GetObjectMeta().GetAnnotations()[annotation] +} + +func UpdateMachinePoolAnnotationJSON(lts scope.LaunchTemplateScope, annotation string, content map[string]interface{}) error { + b, err := json.Marshal(content) + if err != nil { + return err + } + + updateMachinePoolAnnotation(lts, annotation, string(b)) + return nil +} + +func updateMachinePoolAnnotation(lts scope.LaunchTemplateScope, annotation, content string) { + // Get the annotations + annotations := lts.GetObjectMeta().GetAnnotations() + + if annotations == nil { + annotations = make(map[string]string) + } + + // Set our annotation to the given content. + annotations[annotation] = content + + // Update the machine object with these annotations + lts.GetObjectMeta().SetAnnotations(annotations) +} + +// tagsChanged determines which tags to delete and which to add. +func tagsChanged(annotation map[string]interface{}, src map[string]string) (bool, map[string]string, map[string]string, map[string]interface{}) { + // Bool tracking if we found any changed state. + changed := false + + // Tracking for created/updated + created := map[string]string{} + + // Tracking for tags that were deleted. + deleted := map[string]string{} + + // The new annotation that we need to set if anything is created/updated. + newAnnotation := map[string]interface{}{} + + // Loop over annotation, checking if entries are in src. + // If an entry is present in annotation but not src, it has been deleted + // since last time. We flag this in the deleted map. + for t, v := range annotation { + _, ok := src[t] + + // Entry isn't in src, it has been deleted. + if !ok { + // Cast v to a string here. This should be fine, tags are always + // strings. + deleted[t] = v.(string) + changed = true + } + } + + // Loop over src, checking for entries in annotation. + // + // If an entry is in src, but not annotation, it has been created since + // last time. + // + // If an entry is in both src and annotation, we compare their values, if + // the value in src differs from that in annotation, the tag has been + // updated since last time. + for t, v := range src { + av, ok := annotation[t] + + // Entries in the src always need to be noted in the newAnnotation. We + // know they're going to be created or updated. + newAnnotation[t] = v + + // Entry isn't in annotation, it's new. + if !ok { + created[t] = v + newAnnotation[t] = v + changed = true + continue + } + + // Entry is in annotation, has the value changed? + if v != av { + created[t] = v + changed = true + } + + // Entry existed in both src and annotation, and their values were + // equal. Nothing to do. + } + + // We made it through the loop, and everything that was in src, was also + // in dst. Nothing changed. + return changed, created, deleted, newAnnotation +} + // GetLaunchTemplate returns the existing LaunchTemplate or nothing if it doesn't exist. // For now by name until we need the input to be something different. func (s *Service) GetLaunchTemplate(launchTemplateName string) (*expinfrav1.AWSLaunchTemplate, string, error) { @@ -93,7 +375,7 @@ func (s *Service) GetLaunchTemplateID(launchTemplateName string) (string, error) } // CreateLaunchTemplate generates a launch template to be used with the autoscaling group. -func (s *Service) CreateLaunchTemplate(scope *scope.MachinePoolScope, imageID *string, userData []byte) (string, error) { +func (s *Service) CreateLaunchTemplate(scope scope.LaunchTemplateScope, imageID *string, userData []byte) (string, error) { s.scope.Info("Create a new launch template") launchTemplateData, err := s.createLaunchTemplateData(scope, imageID, userData) @@ -103,7 +385,7 @@ func (s *Service) CreateLaunchTemplate(scope *scope.MachinePoolScope, imageID *s input := &ec2.CreateLaunchTemplateInput{ LaunchTemplateData: launchTemplateData, - LaunchTemplateName: aws.String(scope.Name()), + LaunchTemplateName: aws.String(scope.LaunchTemplateName()), } additionalTags := scope.AdditionalTags() @@ -111,9 +393,9 @@ func (s *Service) CreateLaunchTemplate(scope *scope.MachinePoolScope, imageID *s additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name())] = string(infrav1.ResourceLifecycleOwned) tags := infrav1.Build(infrav1.BuildParams{ - ClusterName: s.scope.Name(), + ClusterName: s.scope.KubernetesClusterName(), Lifecycle: infrav1.ResourceLifecycleOwned, - Name: aws.String(scope.Name()), + Name: aws.String(scope.LaunchTemplateName()), Role: aws.String("node"), Additional: additionalTags, }) @@ -137,8 +419,8 @@ func (s *Service) CreateLaunchTemplate(scope *scope.MachinePoolScope, imageID *s } // CreateLaunchTemplateVersion will create a launch template. -func (s *Service) CreateLaunchTemplateVersion(scope *scope.MachinePoolScope, imageID *string, userData []byte) error { - s.scope.V(2).Info("creating new launch template version", "machine-pool", scope.Name()) +func (s *Service) CreateLaunchTemplateVersion(id string, scope scope.LaunchTemplateScope, imageID *string, userData []byte) error { + s.scope.V(2).Info("creating new launch template version", "machine-pool", scope.LaunchTemplateName()) launchTemplateData, err := s.createLaunchTemplateData(scope, imageID, userData) if err != nil { @@ -147,7 +429,7 @@ func (s *Service) CreateLaunchTemplateVersion(scope *scope.MachinePoolScope, ima input := &ec2.CreateLaunchTemplateVersionInput{ LaunchTemplateData: launchTemplateData, - LaunchTemplateId: aws.String(scope.AWSMachinePool.Status.LaunchTemplateID), + LaunchTemplateId: &id, } _, err = s.EC2Client.CreateLaunchTemplateVersion(input) @@ -158,8 +440,8 @@ func (s *Service) CreateLaunchTemplateVersion(scope *scope.MachinePoolScope, ima return nil } -func (s *Service) createLaunchTemplateData(scope *scope.MachinePoolScope, imageID *string, userData []byte) (*ec2.RequestLaunchTemplateData, error) { - lt := scope.AWSMachinePool.Spec.AWSLaunchTemplate +func (s *Service) createLaunchTemplateData(scope scope.LaunchTemplateScope, imageID *string, userData []byte) (*ec2.RequestLaunchTemplateData, error) { + lt := scope.GetLaunchTemplate() // An explicit empty string for SSHKeyName means do not specify a key in the ASG launch var sshKeyNamePtr *string @@ -169,11 +451,25 @@ func (s *Service) createLaunchTemplateData(scope *scope.MachinePoolScope, imageI data := &ec2.RequestLaunchTemplateData{ InstanceType: aws.String(lt.InstanceType), - IamInstanceProfile: &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{ + KeyName: sshKeyNamePtr, + UserData: pointer.StringPtr(base64.StdEncoding.EncodeToString(userData)), + } + + if len(lt.IamInstanceProfile) > 0 { + data.IamInstanceProfile = &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{ Name: aws.String(lt.IamInstanceProfile), - }, - KeyName: sshKeyNamePtr, - UserData: pointer.StringPtr(base64.StdEncoding.EncodeToString(userData)), + } + } + if s.scope.VPC().EnableIPv6 { + data.NetworkInterfaces = []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ + { + Ipv6Prefixes: []*ec2.Ipv6PrefixSpecificationRequest{ + { + Ipv6Prefix: aws.String("auto"), + }, + }, + }, + } } ids, err := s.GetCoreNodeSecurityGroups(scope) @@ -186,7 +482,7 @@ func (s *Service) createLaunchTemplateData(scope *scope.MachinePoolScope, imageI } // add additional security groups as well - securityGroupIDs, err := s.GetAdditionalSecurityGroupsIDs(scope.AWSMachinePool.Spec.AWSLaunchTemplate.AdditionalSecurityGroups) + securityGroupIDs, err := s.GetAdditionalSecurityGroupsIDs(scope.GetLaunchTemplate().AdditionalSecurityGroups) if err != nil { return nil, err } @@ -296,6 +592,25 @@ func (s *Service) PruneLaunchTemplateVersions(id string) error { return s.deleteLaunchTemplateVersion(id, versionToPrune) } +func (s *Service) GetLaunchTemplateLatestVersion(id string) (string, error) { + input := &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateId: aws.String(id), + Versions: aws.StringSlice([]string{expinfrav1.LaunchTemplateLatestVersion}), + } + + out, err := s.EC2Client.DescribeLaunchTemplateVersions(input) + if err != nil { + s.scope.Info("", "aerr", err.Error()) + return "", err + } + + if len(out.LaunchTemplateVersions) == 0 { + return "", errors.Wrapf(err, "failed to get latest launch template version %q", id) + } + + return strconv.Itoa(int(*out.LaunchTemplateVersions[0].VersionNumber)), nil +} + func (s *Service) deleteLaunchTemplateVersion(id string, version *int64) error { s.scope.V(2).Info("Deleting launch template version", "id", id) @@ -326,10 +641,13 @@ func (s *Service) SDKToLaunchTemplate(d *ec2.LaunchTemplateVersion) (*expinfrav1 AMI: infrav1.AMIReference{ ID: v.ImageId, }, - IamInstanceProfile: aws.StringValue(v.IamInstanceProfile.Name), - InstanceType: aws.StringValue(v.InstanceType), - SSHKeyName: v.KeyName, - VersionNumber: d.VersionNumber, + InstanceType: aws.StringValue(v.InstanceType), + SSHKeyName: v.KeyName, + VersionNumber: d.VersionNumber, + } + + if v.IamInstanceProfile != nil { + i.IamInstanceProfile = aws.StringValue(v.IamInstanceProfile.Name) } // Extract IAM Instance Profile name from ARN @@ -362,7 +680,7 @@ func (s *Service) SDKToLaunchTemplate(d *ec2.LaunchTemplateVersion) (*expinfrav1 // // FIXME(dlipovetsky): This check should account for changed userdata, but does not yet do so. // Although userdata is stored in an EC2 Launch Template, it is not a field of AWSLaunchTemplate. -func (s *Service) LaunchTemplateNeedsUpdate(scope *scope.MachinePoolScope, incoming *expinfrav1.AWSLaunchTemplate, existing *expinfrav1.AWSLaunchTemplate) (bool, error) { +func (s *Service) LaunchTemplateNeedsUpdate(scope scope.LaunchTemplateScope, incoming *expinfrav1.AWSLaunchTemplate, existing *expinfrav1.AWSLaunchTemplate) (bool, error) { if incoming.IamInstanceProfile != existing.IamInstanceProfile { return true, nil } @@ -397,14 +715,14 @@ func (s *Service) LaunchTemplateNeedsUpdate(scope *scope.MachinePoolScope, incom } // DiscoverLaunchTemplateAMI will discover the AMI launch template. -func (s *Service) DiscoverLaunchTemplateAMI(scope *scope.MachinePoolScope) (*string, error) { - lt := scope.AWSMachinePool.Spec.AWSLaunchTemplate +func (s *Service) DiscoverLaunchTemplateAMI(scope scope.LaunchTemplateScope) (*string, error) { + lt := scope.GetLaunchTemplate() if lt.AMI.ID != nil { return lt.AMI.ID, nil } - if scope.MachinePool.Spec.Template.Spec.Version == nil { + if scope.GetMachinePool().Spec.Template.Spec.Version == nil { err := errors.New("Either AWSMachinePool's spec.awslaunchtemplate.ami.id or MachinePool's spec.template.spec.version must be defined") s.scope.Error(err, "") return nil, err @@ -415,26 +733,26 @@ func (s *Service) DiscoverLaunchTemplateAMI(scope *scope.MachinePoolScope) (*str imageLookupFormat := lt.ImageLookupFormat if imageLookupFormat == "" { - imageLookupFormat = scope.InfraCluster.ImageLookupFormat() + imageLookupFormat = scope.GetEC2Scope().ImageLookupFormat() } imageLookupOrg := lt.ImageLookupOrg if imageLookupOrg == "" { - imageLookupOrg = scope.InfraCluster.ImageLookupOrg() + imageLookupOrg = scope.GetEC2Scope().ImageLookupOrg() } imageLookupBaseOS := lt.ImageLookupBaseOS if imageLookupBaseOS == "" { - imageLookupBaseOS = scope.InfraCluster.ImageLookupBaseOS() + imageLookupBaseOS = scope.GetEC2Scope().ImageLookupBaseOS() } if scope.IsEKSManaged() && imageLookupFormat == "" && imageLookupOrg == "" && imageLookupBaseOS == "" { - lookupAMI, err = s.eksAMILookup(*scope.MachinePool.Spec.Template.Spec.Version, scope.AWSMachinePool.Spec.AWSLaunchTemplate.AMI.EKSOptimizedLookupType) + lookupAMI, err = s.eksAMILookup(*scope.GetMachinePool().Spec.Template.Spec.Version, scope.GetLaunchTemplate().AMI.EKSOptimizedLookupType) if err != nil { return nil, err } } else { - lookupAMI, err = s.defaultAMIIDLookup(imageLookupFormat, imageLookupOrg, imageLookupBaseOS, *scope.MachinePool.Spec.Template.Spec.Version) + lookupAMI, err = s.defaultAMIIDLookup(imageLookupFormat, imageLookupOrg, imageLookupBaseOS, *scope.GetMachinePool().Spec.Template.Spec.Version) if err != nil { return nil, err } @@ -462,16 +780,16 @@ func (s *Service) GetAdditionalSecurityGroupsIDs(securityGroups []infrav1.AWSRes return additionalSecurityGroupsIDs, nil } -func (s *Service) buildLaunchTemplateTagSpecificationRequest(scope *scope.MachinePoolScope) []*ec2.LaunchTemplateTagSpecificationRequest { +func (s *Service) buildLaunchTemplateTagSpecificationRequest(scope scope.LaunchTemplateScope) []*ec2.LaunchTemplateTagSpecificationRequest { tagSpecifications := make([]*ec2.LaunchTemplateTagSpecificationRequest, 0) additionalTags := scope.AdditionalTags() // Set the cloud provider tag - additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name())] = string(infrav1.ResourceLifecycleOwned) + additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName())] = string(infrav1.ResourceLifecycleOwned) tags := infrav1.Build(infrav1.BuildParams{ - ClusterName: s.scope.Name(), + ClusterName: s.scope.KubernetesClusterName(), Lifecycle: infrav1.ResourceLifecycleOwned, - Name: aws.String(scope.Name()), + Name: aws.String(scope.LaunchTemplateName()), Role: aws.String("node"), Additional: additionalTags, }) diff --git a/pkg/cloud/services/ec2/launchtemplate_test.go b/pkg/cloud/services/ec2/launchtemplate_test.go index d1b30296f7..846b9ad10e 100644 --- a/pkg/cloud/services/ec2/launchtemplate_test.go +++ b/pkg/cloud/services/ec2/launchtemplate_test.go @@ -1053,10 +1053,10 @@ func TestCreateLaunchTemplateVersion(t *testing.T) { cs, err := setupClusterScope(client) g.Expect(err).NotTo(HaveOccurred()) - mpScope, err := setupMachinePoolScope(client, cs) + ms, err := setupMachinePoolScope(client, cs) g.Expect(err).NotTo(HaveOccurred()) - mpScope.AWSMachinePool.Spec.AWSLaunchTemplate.AdditionalSecurityGroups = tc.awsResourceReference + ms.AWSMachinePool.Spec.AWSLaunchTemplate.AdditionalSecurityGroups = tc.awsResourceReference mockEC2Client := mocks.NewMockEC2API(mockCtrl) s := NewService(cs) @@ -1066,10 +1066,10 @@ func TestCreateLaunchTemplateVersion(t *testing.T) { tc.expect(mockEC2Client.EXPECT()) } if tc.wantErr { - g.Expect(s.CreateLaunchTemplateVersion(mpScope, aws.String("imageID"), userData)).To(HaveOccurred()) + g.Expect(s.CreateLaunchTemplateVersion("launch-template-id", ms, aws.String("imageID"), userData)).To(HaveOccurred()) return } - g.Expect(s.CreateLaunchTemplateVersion(mpScope, aws.String("imageID"), userData)).NotTo(HaveOccurred()) + g.Expect(s.CreateLaunchTemplateVersion("launch-template-id", ms, aws.String("imageID"), userData)).NotTo(HaveOccurred()) }) } } @@ -1114,11 +1114,11 @@ func TestBuildLaunchTemplateTagSpecificationRequest(t *testing.T) { cs, err := setupClusterScope(client) g.Expect(err).NotTo(HaveOccurred()) - mpScope, err := setupMachinePoolScope(client, cs) + ms, err := setupMachinePoolScope(client, cs) g.Expect(err).NotTo(HaveOccurred()) s := NewService(cs) - tc.check(g, s.buildLaunchTemplateTagSpecificationRequest(mpScope)) + tc.check(g, s.buildLaunchTemplateTagSpecificationRequest(ms)) }) } } diff --git a/pkg/cloud/services/eks/cluster.go b/pkg/cloud/services/eks/cluster.go index 00afedae1d..c4604bd57d 100644 --- a/pkg/cloud/services/eks/cluster.go +++ b/pkg/cloud/services/eks/cluster.go @@ -59,10 +59,16 @@ func (s *Service) reconcileCluster(ctx context.Context) error { return errors.Wrap(err, "failed to create cluster") } } else { - tagKey := infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName()) + tagKey := infrav1.ClusterAWSCloudProviderTagKey(eksClusterName) ownedTag := cluster.Tags[tagKey] - if ownedTag == nil { - return fmt.Errorf("checking owner of %s is %s: %w", s.scope.KubernetesClusterName(), s.scope.Name(), err) + // Prior to https://github.com/kubernetes-sigs/cluster-api-provider-aws/pull/3573, + // Clusters were tagged using s.scope.Name() + // To support upgrading older clusters, check for both tags + oldTagKey := infrav1.ClusterAWSCloudProviderTagKey(s.scope.Name()) + oldOwnedTag := cluster.Tags[oldTagKey] + + if ownedTag == nil && oldOwnedTag == nil { + return fmt.Errorf("checking owner of %s is %s: %w", s.scope.KubernetesClusterName(), eksClusterName, err) } s.scope.V(2).Info("Found owned EKS cluster in AWS", "cluster-name", eksClusterName) @@ -223,7 +229,7 @@ func makeEksEncryptionConfigs(encryptionConfig *ekscontrolplanev1.EncryptionConf if encryptionConfig == nil { return cfg } - //TODO: change EncryptionConfig so that provider and resources are required if encruptionConfig is specified + // TODO: change EncryptionConfig so that provider and resources are required if encruptionConfig is specified if encryptionConfig.Provider == nil || len(*encryptionConfig.Provider) == 0 { return cfg } @@ -304,8 +310,8 @@ func makeEksLogging(loggingSpec *ekscontrolplanev1.ControlPlaneLoggingSpec) *eks if loggingSpec == nil { return nil } - var on = true - var off = false + on := true + off := false var enabledTypes []string var disabledTypes []string @@ -353,9 +359,17 @@ func (s *Service) createCluster(eksClusterName string) (*eks.Cluster, error) { if err != nil { return nil, errors.Wrap(err, "couldn't create vpc config for cluster") } - netConfig, err := makeKubernetesNetworkConfig(s.scope.ServiceCidrs()) - if err != nil { - return nil, errors.Wrap(err, "couldn't create Kubernetes network config for cluster") + + var netConfig *eks.KubernetesNetworkConfigRequest + if s.scope.VPC().EnableIPv6 { + netConfig = &eks.KubernetesNetworkConfigRequest{ + IpFamily: aws.String(eks.IpFamilyIpv6), + } + } else { + netConfig, err = makeKubernetesNetworkConfig(s.scope.ServiceCidrs()) + if err != nil { + return nil, errors.Wrap(err, "couldn't create Kubernetes network config for cluster") + } } // Make sure to use the MachineScope here to get the merger of AWSCluster and AWSMachine tags @@ -397,7 +411,7 @@ func (s *Service) createCluster(eksClusterName string) (*eks.Cluster, error) { conditions.MarkTrue(s.scope.ControlPlane, ekscontrolplanev1.EKSControlPlaneCreatingCondition) record.Eventf(s.scope.ControlPlane, "InitiatedCreateEKSControlPlane", "Initiated creation of a new EKS control plane %s", s.scope.KubernetesClusterName()) return true, nil - }, awserrors.ResourceNotFound); err != nil { //TODO: change the error that can be retried + }, awserrors.ResourceNotFound); err != nil { // TODO: change the error that can be retried record.Warnf(s.scope.ControlPlane, "FailedCreateEKSControlPlane", "Failed to initiate creation of a new EKS control plane: %v", err) return nil, errors.Wrapf(err, "failed to create EKS cluster") } diff --git a/pkg/cloud/services/eks/cluster_test.go b/pkg/cloud/services/eks/cluster_test.go index 0d9dcc2425..ae003a01c7 100644 --- a/pkg/cloud/services/eks/cluster_test.go +++ b/pkg/cloud/services/eks/cluster_test.go @@ -179,6 +179,33 @@ func TestMakeVPCConfig(t *testing.T) { SubnetIds: []*string{&idOne, &idTwo}, }, }, + { + name: "ipv6 subnets", + input: input{ + subnets: []infrav1.SubnetSpec{ + { + ID: idOne, + CidrBlock: "10.0.10.0/24", + AvailabilityZone: "us-west-2a", + IsPublic: true, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:85a3:1::/64", + }, + { + ID: idTwo, + CidrBlock: "10.0.10.0/24", + AvailabilityZone: "us-west-2b", + IsPublic: false, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:85a3:2::/64", + }, + }, + endpointAccess: ekscontrolplanev1.EndpointAccess{}, + }, + expect: &eks.VpcConfigRequest{ + SubnetIds: []*string{&idOne, &idTwo}, + }, + }, { name: "security groups", input: input{ @@ -642,3 +669,100 @@ func TestReconcileEKSEncryptionConfig(t *testing.T) { }) } } + +func TestCreateIPv6Cluster(t *testing.T) { + g := NewWithT(t) + + mockControl := gomock.NewController(t) + defer mockControl.Finish() + + eksMock := mock_eksiface.NewMockEKSAPI(mockControl) + iamMock := mock_iamauth.NewMockIAMAPI(mockControl) + + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + _ = ekscontrolplanev1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + encryptionConfig := &ekscontrolplanev1.EncryptionConfig{ + Provider: pointer.String("new-provider"), + Resources: []*string{pointer.String("foo"), pointer.String("bar")}, + } + vpcSpec := infrav1.VPCSpec{ + IPv6CidrBlock: "2001:db8:85a3::/56", + EnableIPv6: true, + } + scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns", + Name: "cluster-name", + }, + }, + ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{ + Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{ + RoleName: pointer.String("arn-role"), + Version: aws.String("1.22"), + NetworkSpec: infrav1.NetworkSpec{ + Subnets: []infrav1.SubnetSpec{ + { + ID: "sub-1", + CidrBlock: "10.0.10.0/24", + AvailabilityZone: "us-west-2a", + IsPublic: true, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:85a3:1::/64", + }, + { + ID: "sub-2", + CidrBlock: "10.0.10.0/24", + AvailabilityZone: "us-west-2b", + IsPublic: false, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:85a3:2::/64", + }, + }, + VPC: vpcSpec, + }, + EncryptionConfig: encryptionConfig, + }, + }, + }) + g.Expect(err).To(BeNil()) + + eksMock.EXPECT().CreateCluster(&eks.CreateClusterInput{ + Name: aws.String("cluster-name"), + Version: aws.String("1.22"), + EncryptionConfig: []*eks.EncryptionConfig{ + { + Provider: &eks.Provider{ + KeyArn: encryptionConfig.Provider, + }, + Resources: encryptionConfig.Resources, + }, + }, + ResourcesVpcConfig: &eks.VpcConfigRequest{ + SubnetIds: []*string{pointer.StringPtr("sub-1"), pointer.StringPtr("sub-2")}, + }, + KubernetesNetworkConfig: &eks.KubernetesNetworkConfigRequest{ + IpFamily: pointer.StringPtr("ipv6"), + }, + Tags: map[string]*string{ + "kubernetes.io/cluster/": pointer.StringPtr("owned"), + }, + }).Return(&eks.CreateClusterOutput{}, nil) + iamMock.EXPECT().GetRole(&iam.GetRoleInput{ + RoleName: aws.String("arn-role"), + }).Return(&iam.GetRoleOutput{ + Role: &iam.Role{ + RoleName: pointer.String("arn-role"), + }, + }, nil) + + s := NewService(scope) + s.EKSClient = eksMock + s.IAMClient = iamMock + + _, err = s.createCluster("cluster-name") + g.Expect(err).To(BeNil()) +} diff --git a/pkg/cloud/services/eks/config.go b/pkg/cloud/services/eks/config.go index 44b7f5759d..83a3d23003 100644 --- a/pkg/cloud/services/eks/config.go +++ b/pkg/cloud/services/eks/config.go @@ -85,7 +85,7 @@ func (s *Service) reconcileAdditionalKubeconfigs(ctx context.Context, cluster *e } // Create the additional kubeconfig for users. This doesn't need updating on every sync - _, err := secret.GetFromNamespacedName(ctx, s.scope.Client, clusterRef, secret.Kubeconfig) + configSecret, err := secret.GetFromNamespacedName(ctx, s.scope.Client, clusterRef, secret.Kubeconfig) if err != nil { if !apierrors.IsNotFound(err) { return errors.Wrap(err, "failed to get kubeconfig (user) secret") @@ -99,6 +99,8 @@ func (s *Service) reconcileAdditionalKubeconfigs(ctx context.Context, cluster *e if createErr != nil { return err } + } else if updateErr := s.updateUserKubeconfigSecret(ctx, configSecret, cluster); updateErr != nil { + return fmt.Errorf("updating kubeconfig secret: %w", err) } return nil @@ -187,7 +189,11 @@ func (s *Service) createUserKubeconfigSecret(ctx context.Context, cluster *eks.C return fmt.Errorf("creating base kubeconfig: %w", err) } - execConfig := &api.ExecConfig{APIVersion: "client.authentication.k8s.io/v1alpha1"} + // Version v1alpha1 was removed in Kubernetes v1.23. + // Version v1 was released in Kubernetes v1.23. + // Version v1beta1 was selected as it has the widest range of support + // This should be changed to v1 once EKS no longer supports Kubernetes 1.15 and then 1.15 -> 1.16. input.Version = aws.String(versionToEKS(ngVersion.WithMinor(ngVersion.Minor() + 1))) updateMsg = fmt.Sprintf("to version %s", *input.Version) - } else if specAMI != nil && *specAMI != ngAMI { + case specAMI != nil && *specAMI != ngAMI: input.ReleaseVersion = specAMI updateMsg = fmt.Sprintf("to AMI version %s", *input.ReleaseVersion) + case statusLaunchTemplateVersion != nil && *statusLaunchTemplateVersion != *ngLaunchTemplateVersion: + input.LaunchTemplate = &eks.LaunchTemplateSpecification{ + Id: s.scope.ManagedMachinePool.Status.LaunchTemplateID, + Version: statusLaunchTemplateVersion, + } + updateMsg = fmt.Sprintf("to launch template version %s", *statusLaunchTemplateVersion) } if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { @@ -458,6 +481,10 @@ func (s *NodegroupService) reconcileNodegroupConfig(ng *eks.Nodegroup) error { (aws.Int64Value(ng.ScalingConfig.MinSize) != int64(aws.Int32Value(managedPool.Scaling.MinSize)))) { s.V(2).Info("Nodegroup min/max differ from spec, updating scaling configuration", "nodegroup", ng.NodegroupName) input.ScalingConfig = s.scalingConfig() + if *ng.ScalingConfig.DesiredSize < int64(aws.Int32Value(managedPool.Scaling.MinSize)) { + desiredSize := int64(aws.Int32Value(managedPool.Scaling.MinSize)) + input.ScalingConfig.DesiredSize = &desiredSize + } needsUpdate = true } currentUpdateConfig := converters.NodegroupUpdateconfigFromSDK(ng.UpdateConfig) @@ -514,6 +541,35 @@ func (s *NodegroupService) reconcileNodegroup() error { break } + if _, ok := s.scope.ManagedMachinePool.GetAnnotations()[AutoscalerEnabled]; ok { + s.scope.V(2).Info("Autoscaler enabled: annotation found") + // Set MachinePool replicas to the ASG DesiredCapacity + req := autoscaling.DescribeAutoScalingGroupsInput{} + for _, asg := range ng.Resources.AutoScalingGroups { + req.AutoScalingGroupNames = append(req.AutoScalingGroupNames, asg.Name) + } + groups, err := s.AutoscalingClient.DescribeAutoScalingGroups(&req) + if err != nil { + return errors.Wrap(err, "failed to describe AutoScalingGroup for nodegroup") + } + + var desiredSize int64 + for _, group := range groups.AutoScalingGroups { + desiredSize += *group.DesiredCapacity + } + + mp := s.scope.MachinePool.DeepCopy() + if *s.scope.MachinePool.Spec.Replicas != int32(desiredSize) { + s.scope.V(2).Info("Autoscaling enabled. Setting MachinePool replicas to ASG DesiredCapacity", + "current", s.scope.MachinePool.Spec.Replicas, + "desired", desiredSize) + s.scope.MachinePool.Spec.Replicas = aws.Int32(int32(desiredSize)) + if err := s.scope.Client.Patch(context.Background(), s.scope.MachinePool, client.MergeFrom(mp)); err != nil { + return errors.Wrapf(err, "failed to patch machine pool with replicas: %v", desiredSize) + } + } + } + if err != nil { return errors.Wrap(err, "failed to wait for nodegroup to be active") } @@ -573,12 +629,7 @@ func (s *NodegroupService) setStatus(ng *eks.Nodegroup) error { for _, group := range groups.AutoScalingGroups { replicas += int32(len(group.Instances)) for _, instance := range group.Instances { - id, err := noderefutil.NewProviderID(fmt.Sprintf("aws://%s/%s", *instance.AvailabilityZone, *instance.InstanceId)) - if err != nil { - s.Error(err, "couldn't create provider ID for instance", "id", *instance.InstanceId) - continue - } - providerIDList = append(providerIDList, id.String()) + providerIDList = append(providerIDList, fmt.Sprintf("aws:///%s/%s", *instance.AvailabilityZone, *instance.InstanceId)) } } managedPool.Spec.ProviderIDList = providerIDList diff --git a/pkg/cloud/services/eks/oidc.go b/pkg/cloud/services/eks/oidc.go index 0aaa8d2b1b..ae35fcd962 100644 --- a/pkg/cloud/services/eks/oidc.go +++ b/pkg/cloud/services/eks/oidc.go @@ -19,31 +19,25 @@ package eks import ( "context" "fmt" - "regexp" "strings" "github.com/aws/aws-sdk-go/service/eks" + awsiam "github.com/aws/aws-sdk-go/service/iam" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/converters" iamv1 "sigs.k8s.io/cluster-api-provider-aws/iam/api/v1beta1" + tagConverter "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/iam" "sigs.k8s.io/cluster-api/controllers/remote" ) -var ( - trustPolicyConfigMapName = "boilerplate-oidc-trust-policy" - trustPolicyConfigMapNamespace = metav1.NamespaceDefault - - whitespaceRe = regexp.MustCompile(`(?m)[\t\n]`) -) - func (s *Service) reconcileOIDCProvider(cluster *eks.Cluster) error { - if !s.scope.ControlPlane.Spec.AssociateOIDCProvider || s.scope.ControlPlane.Status.OIDCProvider.ARN != "" { + if !s.scope.ControlPlane.Spec.AssociateOIDCProvider { return nil } @@ -51,24 +45,39 @@ func (s *Service) reconcileOIDCProvider(cluster *eks.Cluster) error { return errors.New("'AssociateOIDCProvider' provided without enabling the 'EKSEnableIAM' feature flag") } - s.scope.Info("Reconciling EKS OIDC Provider", "cluster-name", cluster.Name) - oidcProvider, err := s.CreateOIDCProvider(cluster) - if err != nil { - return errors.Wrap(err, "failed to create OIDC provider") + if s.scope.ControlPlane.Status.OIDCProvider.ARN == "" { + s.scope.Info("Reconciling EKS OIDC Provider", "cluster-name", cluster.Name) + oidcProvider, err := s.CreateOIDCProvider(cluster) + if err != nil { + return errors.Wrap(err, "failed to create OIDC provider") + } + s.scope.ControlPlane.Status.OIDCProvider.ARN = oidcProvider + if err := s.scope.PatchObject(); err != nil { + return errors.Wrap(err, "failed to update control plane with OIDC provider ARN") + } } - s.scope.ControlPlane.Status.OIDCProvider.ARN = oidcProvider - policy, err := converters.IAMPolicyDocumentToJSON(s.buildOIDCTrustPolicy()) - if err != nil { - return errors.Wrap(err, "failed to parse IAM policy") + // tagging the OIDC provider with the same tags of cluster + inputForTags := awsiam.TagOpenIDConnectProviderInput{ + OpenIDConnectProviderArn: &s.scope.ControlPlane.Status.OIDCProvider.ARN, + Tags: tagConverter.MapToIAMTags(tagConverter.MapPtrToMap(cluster.Tags)), } - s.scope.ControlPlane.Status.OIDCProvider.TrustPolicy = whitespaceRe.ReplaceAllString(policy, "") - if err := s.scope.PatchObject(); err != nil { - return errors.Wrap(err, "failed to update control plane with OIDC provider ARN") + if _, err := s.IAMClient.TagOpenIDConnectProvider(&inputForTags); err != nil { + return errors.Wrap(err, "failed to tag OIDC provider") } - if err := s.reconcileTrustPolicy(); err != nil { - return errors.Wrap(err, "failed to reconcile trust policy in workload cluster") + if s.scope.ControlPlane.Status.OIDCProvider.TrustPolicy == "" { + policy, err := converters.IAMPolicyDocumentToJSON(s.buildOIDCTrustPolicy()) + if err != nil { + return errors.Wrap(err, "failed to parse IAM policy") + } + if err := s.reconcileTrustPolicy(); err != nil { + return errors.Wrap(err, "failed to reconcile trust policy in workload cluster") + } + s.scope.ControlPlane.Status.OIDCProvider.TrustPolicy = iam.WhitespaceRegex.ReplaceAllString(policy, "") + if err := s.scope.PatchObject(); err != nil { + return errors.Wrap(err, "failed to update control plane with OIDC provider trustPolicy") + } } return nil @@ -93,15 +102,15 @@ func (s *Service) reconcileTrustPolicy() error { } configMapRef := types.NamespacedName{ - Name: trustPolicyConfigMapName, - Namespace: trustPolicyConfigMapNamespace, + Name: iam.TrustPolicyConfigMapName, + Namespace: iam.TrustPolicyConfigMapNamespace, } trustPolicyConfigMap := &corev1.ConfigMap{} err = remoteClient.Get(ctx, configMapRef, trustPolicyConfigMap) if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("getting %s/%s config map: %w", trustPolicyConfigMapNamespace, trustPolicyConfigMapName, err) + return fmt.Errorf("getting %s/%s config map: %w", iam.TrustPolicyConfigMapNamespace, iam.TrustPolicyConfigMapName, err) } policy, err := converters.IAMPolicyDocumentToJSON(s.buildOIDCTrustPolicy()) @@ -114,13 +123,13 @@ func (s *Service) reconcileTrustPolicy() error { } if trustPolicyConfigMap.UID == "" { - trustPolicyConfigMap.Name = trustPolicyConfigMapName - trustPolicyConfigMap.Namespace = trustPolicyConfigMapNamespace - s.V(2).Info("Creating new Trust Policy ConfigMap", "cluster", s.scope.Name(), "configmap", trustPolicyConfigMapName) + trustPolicyConfigMap.Name = iam.TrustPolicyConfigMapName + trustPolicyConfigMap.Namespace = iam.TrustPolicyConfigMapNamespace + s.V(2).Info("Creating new Trust Policy ConfigMap", "cluster", s.scope.Name(), "configmap", iam.TrustPolicyConfigMapName) return remoteClient.Create(ctx, trustPolicyConfigMap) } - s.V(2).Info("Updating existing Trust Policy ConfigMap", "cluster", s.scope.Name(), "configmap", trustPolicyConfigMapName) + s.V(2).Info("Updating existing Trust Policy ConfigMap", "cluster", s.scope.Name(), "configmap", iam.TrustPolicyConfigMapName) return remoteClient.Update(ctx, trustPolicyConfigMap) } diff --git a/pkg/cloud/services/eks/roles.go b/pkg/cloud/services/eks/roles.go index 91a73fe4b0..a036ceaaf6 100644 --- a/pkg/cloud/services/eks/roles.go +++ b/pkg/cloud/services/eks/roles.go @@ -18,6 +18,8 @@ package eks import ( "fmt" + "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/api/bootstrap/v1beta1" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -52,6 +54,22 @@ func FargateRolePolicies() []string { } } +// NodegroupRolePoliciesUSGov gives the policies required for a nodegroup role. +func NodegroupRolePoliciesUSGov() []string { + return []string{ + "arn:aws-us-gov:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws-us-gov:iam::aws:policy/AmazonEKS_CNI_Policy", //TODO: Can remove when CAPA supports provisioning of OIDC web identity federation with service account token volume projection + "arn:aws-us-gov:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + } +} + +// FargateRolePoliciesUSGov gives the policies required for a fargate role. +func FargateRolePoliciesUSGov() []string { + return []string{ + "arn:aws-us-gov:iam::aws:policy/AmazonEKSFargatePodExecutionRolePolicy", + } +} + func (s *Service) reconcileControlPlaneIAMRole() error { s.scope.V(2).Info("Reconciling EKS Control Plane IAM Role") @@ -92,10 +110,10 @@ func (s *Service) reconcileControlPlaneIAMRole() error { } //TODO: check tags and trust relationship to see if they need updating - policies := []*string{ - aws.String("arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"), + aws.String(fmt.Sprintf("arn:%s:iam::aws:policy/AmazonEKSClusterPolicy", s.scope.Partition())), } + if s.scope.ControlPlane.Spec.RoleAdditionalPolicies != nil { if !s.scope.AllowAdditionalRoles() && len(*s.scope.ControlPlane.Spec.RoleAdditionalPolicies) > 0 { return ErrCannotUseAdditionalRoles @@ -204,6 +222,9 @@ func (s *NodegroupService) reconcileNodegroupIAMRole() error { } policies := NodegroupRolePolicies() + if strings.Contains(s.scope.Partition(), v1beta1.PartitionNameUSGov) { + policies = NodegroupRolePoliciesUSGov() + } if len(s.scope.ManagedMachinePool.Spec.RoleAdditionalPolicies) > 0 { if !s.scope.AllowAdditionalRoles() { return ErrCannotUseAdditionalRoles @@ -320,6 +341,9 @@ func (s *FargateService) reconcileFargateIAMRole() (requeue bool, err error) { } policies := FargateRolePolicies() + if strings.Contains(s.scope.Partition(), v1beta1.PartitionNameUSGov) { + policies = FargateRolePoliciesUSGov() + } updatedPolicies, err := s.EnsurePoliciesAttached(role, aws.StringSlice(policies)) if err != nil { return updatedRole, errors.Wrapf(err, "error ensuring policies are attached: %v", policies) diff --git a/pkg/cloud/services/eks/tags.go b/pkg/cloud/services/eks/tags.go index e27e474b67..290a1eaf00 100644 --- a/pkg/cloud/services/eks/tags.go +++ b/pkg/cloud/services/eks/tags.go @@ -52,7 +52,7 @@ func (s *Service) getEKSTagParams(id string) *infrav1.BuildParams { name := s.scope.KubernetesClusterName() return &infrav1.BuildParams{ - ClusterName: s.scope.Name(), + ClusterName: name, ResourceID: id, Lifecycle: infrav1.ResourceLifecycleOwned, Name: aws.String(name), diff --git a/pkg/cloud/services/iam/iam.go b/pkg/cloud/services/iam/iam.go new file mode 100644 index 0000000000..dc9c9a6387 --- /dev/null +++ b/pkg/cloud/services/iam/iam.go @@ -0,0 +1,135 @@ +package iam + +import ( + "context" + "errors" + "regexp" + + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/aws/aws-sdk-go/service/iam/iamiface" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/s3" +) + +type Service struct { + scope scope.EC2Scope + IAMClient iamiface.IAMAPI +} + +// NewService returns a new service given the api clients. +func NewService(clusterScope scope.EC2Scope) *Service { + iamClient := scope.NewIAMClient(clusterScope, clusterScope, clusterScope, clusterScope.InfraCluster()) + + return &Service{ + scope: clusterScope, + IAMClient: iamClient, + } +} + +const ( + // TrustPolicyJSON is the data key for the configmap containing the oidc trust policy. + TrustPolicyJSON = "trust-policy.json" + + // PodIdentityWebhookCertificateFormat the format for the cert name used for the pod-identity-webhook. + PodIdentityWebhookCertificateFormat = "%s-pod-id-wh" + + // SelfsignedIssuerFormat format for the self signed issuer used for the cluster to make the pod-identity-webhook cert. + SelfsignedIssuerFormat = "%s-selfsigned-issuer" + + // S3HostFormat format for the host format for s3 for the oidc provider. + S3HostFormat = "s3-%s.amazonaws.com" + + // STSAWSAudience security token service url. + STSAWSAudience = "sts.amazonaws.com" + + // TrustPolicyConfigMapName name of the configmap that contains the trust policy template. + TrustPolicyConfigMapName = "boilerplate-oidc-trust-policy" + + // TrustPolicyConfigMapNamespace namespace the trust policy template is put into. + TrustPolicyConfigMapNamespace = metav1.NamespaceDefault +) + +var ( + WhitespaceRegex = regexp.MustCompile(`(?m)[\t\n]`) +) + +// ReconcileOIDCProvider replicates functionality already built into managed clusters by auto-deploying the +// modifying kube-apiserver args, deploying the pod identity webhook and setting/configuring an oidc provider +// for more details see: https://github.com/aws/amazon-eks-pod-identity-webhook/blob/master/SELF_HOSTED_SETUP.md +// 1. create a self signed issuer for the mutating webhook +// 2. add create a json patch for kube-apiserver and use capi config to add to the kubeadm.yml +// 3. create an oidc provider in aws which points to the s3 bucket +// 4. pause until kubeconfig and cluster acccess is ready +// 5. move openid config and jwks to the s3 bucket +// 6. add the pod identity webhook to the workload cluster +// 7. add the configmap to the workload cluster. +func (s *Service) ReconcileOIDCProvider(ctx context.Context) error { + if !s.scope.AssociateOIDCProvider() { + return nil + } + + log := ctrl.LoggerFrom(ctx) + log.Info("Associating OIDC Provider") + + if s.scope.Bucket() == nil { + return errors.New("s3 bucket configuration required to associate oidc provider") + } + + if err := s.reconcileSelfsignedIssuer(ctx); err != nil { + return err + } + + if err := s.reconcileKubeAPIParameters(ctx); err != nil { + return err + } + + if err := s.reconcileIdentityProvider(ctx); err != nil { + return err + } + + // the following can only run with a working workload cluster, return nil until then + _, ok := s.scope.InfraCluster().GetAnnotations()[scope.KubeconfigReadyAnnotation] + if !ok { + log.Info("Associating OIDC Provider paused, kubeconfig and workload cluster API access is not ready") + return nil + } + + log.Info("Associating OIDC Provider continuing, kubeconfig for the workload cluster is available") + if err := s.reconcileBucketContents(ctx); err != nil { + return err + } + + if err := s.reconcilePodIdentityWebhook(ctx); err != nil { + return err + } + + return s.reconcileTrustPolicyConfigMap(ctx) +} + +// DeleteOIDCProvider will delete the iam resources note that the bucket is cleaned up in the s3 service +// 1. delete oidc provider +// 2. delete mwh certificate +// 3. delete cert-manager issuer. +func (s *Service) DeleteOIDCProvider(ctx context.Context) error { + if !s.scope.AssociateOIDCProvider() { + return nil + } + + log := ctrl.LoggerFrom(ctx) + log.Info("Deleting OIDC Provider") + + if s.scope.Bucket() != nil { + if err := deleteBucketContents(s3.NewService(s.scope)); err != nil { + return err + } + } + + if err := deleteCertificatesAndIssuer(ctx, s.scope.Name(), s.scope.Namespace(), s.scope.ManagementClient()); err != nil { + return err + } + + return deleteOIDCProvider(s.scope.OIDCProviderStatus().ARN, s.IAMClient) +} diff --git a/pkg/cloud/services/iam/oidc.go b/pkg/cloud/services/iam/oidc.go new file mode 100644 index 0000000000..01725c0e0c --- /dev/null +++ b/pkg/cloud/services/iam/oidc.go @@ -0,0 +1,375 @@ +package iam + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/tls" + "fmt" + "path" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/iam/iamiface" + v1certmanager "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + v1certmanagermeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + + iamv1 "sigs.k8s.io/cluster-api-provider-aws/iam/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/s3" + "sigs.k8s.io/cluster-api/api/v1beta1" + v1beta12 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" + v1beta13 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" +) + +const ( + jwksKey = "/openid/v1/jwks" + opendIDConfigurationKey = "/.well-known/openid-configuration" +) + +func certificateSecret(ctx context.Context, name, namespace, issuer string, dnsNames []string, client client.Client) (*corev1.Secret, error) { + // check if the secret was already created + certSecret := &corev1.Secret{} + if err := client.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, certSecret); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + if certSecret.UID != "" { + return certSecret, nil + } + + // cert does not exist, create the request + cert := &v1certmanager.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.ProviderLabelName: "infrastructure-aws", + }, + Name: name, + Namespace: namespace, + }, + Spec: v1certmanager.CertificateSpec{ + SecretName: name, + IsCA: true, + PrivateKey: &v1certmanager.CertificatePrivateKey{ + Algorithm: v1certmanager.RSAKeyAlgorithm, + Size: 2048, + }, + IssuerRef: v1certmanagermeta.ObjectReference{ + Kind: "Issuer", + Name: issuer, + }, + DNSNames: dnsNames, + }, + } + + // check if cert already exists + if err := client.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, cert); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + + if cert.UID == "" { + if err := client.Create(ctx, cert); err != nil { + return nil, err + } + } + + // check if the secret was created by cert-manager, return all errors until cert manager is done + return certSecret, client.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, certSecret) +} + +func deleteBucketContents(s3 *s3.Service) error { + if err := s3.Delete(jwksKey); err != nil { + return err + } + + return s3.Delete(opendIDConfigurationKey) +} + +func deleteCertificatesAndIssuer(ctx context.Context, name, namespace string, client client.Client) error { + certs := []string{ + fmt.Sprintf(PodIdentityWebhookCertificateFormat, name), + } + + for _, c := range certs { + cert := &v1certmanager.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: c, + Namespace: namespace, + }, + } + if err := client.Delete(ctx, cert); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + + if err := client.Delete(ctx, &v1certmanager.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf(SelfsignedIssuerFormat, name), + Namespace: namespace, + }, + }); err != nil && !apierrors.IsNotFound(err) { + return err + } + + return nil +} + +func (s *Service) reconcileBucketContents(ctx context.Context) error { + clusterKey := client.ObjectKey{ + Name: s.scope.Name(), + Namespace: s.scope.Namespace(), + } + + // get remote config from management cluster + remoteRestConfig, err := remote.RESTConfig(context.Background(), s.scope.Name(), s.scope.ManagementClient(), clusterKey) + if err != nil { + return fmt.Errorf("getting remote rest config for %s/%s: %w", s.scope.Namespace(), s.scope.Name(), err) + } + remoteRestConfig.Timeout = scope.DefaultKubeClientTimeout + + // make a client set for the workload cluster + clientSet, err := kubernetes.NewForConfig(remoteRestConfig) + if err != nil { + return err + } + + s3scope := s3.NewService(s.scope) + conf, err := get(ctx, clientSet, opendIDConfigurationKey) + if err != nil { + return err + } + + if _, err := s3scope.CreatePublic(opendIDConfigurationKey, []byte(conf)); err != nil { + return err + } + + jwks, err := get(ctx, clientSet, jwksKey) + if err != nil { + return err + } + + if _, err := s3scope.CreatePublic(jwksKey, []byte(jwks)); err != nil { + return err + } + + return nil +} + +func get(ctx context.Context, clientSet *kubernetes.Clientset, uri string) (ret string, err error) { + request := clientSet.RESTClient().Get().RequestURI(uri) + stream, err := request.Stream(ctx) + if err != nil { + return + } + defer func() { + err = stream.Close() + }() + + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(stream) + if err != nil { + return + } + + if err = stream.Close(); err != nil { + return + } + + ret = buf.String() + return +} + +func buildOIDCTrustPolicy(arn string) iamv1.PolicyDocument { + conditionValue := arn[strings.Index(arn, "/")+1:] + ":sub" + + return iamv1.PolicyDocument{ + Version: "2012-10-17", + Statement: iamv1.Statements{ + iamv1.StatementEntry{ + Sid: "", + Effect: "Allow", + Principal: iamv1.Principals{ + iamv1.PrincipalFederated: iamv1.PrincipalID{arn}, + }, + Action: iamv1.Actions{"sts:AssumeRoleWithWebIdentity"}, + Condition: iamv1.Conditions{ + "ForAnyValue:StringLike": map[string][]string{ + conditionValue: {"system:serviceaccount:${SERVICE_ACCOUNT_NAMESPACE}:${SERVICE_ACCOUNT_NAME}"}, + }, + }, + }, + }, + } +} + +// FindAndVerifyOIDCProvider will try to find an OIDC provider. It will return an error if the found provider does not +// match the cluster spec. +func findAndVerifyOIDCProvider(issuerURL, thumbprint string, iamClient iamiface.IAMAPI) (string, error) { + output, err := iamClient.ListOpenIDConnectProviders(&iam.ListOpenIDConnectProvidersInput{}) + if err != nil { + return "", errors.Wrap(err, "error listing providers") + } + for _, r := range output.OpenIDConnectProviderList { + provider, err := iamClient.GetOpenIDConnectProvider(&iam.GetOpenIDConnectProviderInput{OpenIDConnectProviderArn: r.Arn}) + if err != nil { + return "", errors.Wrap(err, "error getting provider") + } + // URL should always contain `https`. + if "https://"+aws.StringValue(provider.Url) != issuerURL { + continue + } + if len(provider.ThumbprintList) != 1 || aws.StringValue(provider.ThumbprintList[0]) != thumbprint { + return "", errors.Wrap(err, "found provider with matching issuerURL but with non-matching thumbprint") + } + if len(provider.ClientIDList) != 1 || aws.StringValue(provider.ClientIDList[0]) != STSAWSAudience { + return "", errors.Wrap(err, "found provider with matching issuerURL but with non-matching clientID") + } + return aws.StringValue(r.Arn), nil + } + return "", nil +} + +func fetchRootCAThumbprint(url string, port int) (ret string, err error) { + // Parse cmdline arguments using flag package + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%d", url, port), &tls.Config{ + MinVersion: tls.VersionTLS12, + }) + if err != nil { + return + } + defer func() { + err = conn.Close() + }() + + // Get the ConnectionState struct as that's the one which gives us x509.Certificate struct + cert := conn.ConnectionState().PeerCertificates[0] + fingerprint := sha1.Sum(cert.Raw) //nolint:gosec // this is not used for real security + var buf bytes.Buffer + for _, f := range fingerprint { + if _, err = fmt.Fprintf(&buf, "%02X", f); err != nil { + return + } + } + ret = strings.ToLower(buf.String()) + return +} + +// DeleteOIDCProvider will delete an OIDC provider. +func deleteOIDCProvider(arn string, iamClient iamiface.IAMAPI) error { + if arn == "" { + return nil + } + + input := iam.DeleteOpenIDConnectProviderInput{ + OpenIDConnectProviderArn: aws.String(arn), + } + + _, err := iamClient.DeleteOpenIDConnectProvider(&input) + if err != nil { + return errors.Wrap(err, "error deleting provider") + } + return nil +} + +// reconcileKubeAPIParameters +// 1. find kubeadmcontrolplane +// 2. use name/namespace to pull kubeadmconfig +// 3. update files/params. +func (s *Service) reconcileKubeAPIParameters(ctx context.Context) error { + managementClient := s.scope.ManagementClient() + name := s.scope.Name() + namespace := s.scope.Namespace() + + s3Host := fmt.Sprintf(S3HostFormat, s.scope.Region()) + accountIssuer := "https://" + path.Join(s3Host, s.scope.Bucket().Name) + + listOptions := []client.ListOption{ + client.InNamespace(namespace), + client.MatchingLabels(map[string]string{v1beta1.ClusterLabelName: name}), + } + + controlPlanes := &v1beta13.KubeadmControlPlaneList{} + if err := managementClient.List(ctx, controlPlanes, listOptions...); err != nil { + return fmt.Errorf("failed to list kubeadm control planes for cluster %s/%s: %w", namespace, name, err) + } + + patchContent := `[ + { + "op": "add", + "path": "/spec/containers/0/command/1", + "value": "--api-audiences=https://kubernetes.default.svc.cluster.local" + }, + { + "op": "add", + "path": "/spec/containers/0/command/1", + "value": "--api-audiences=` + STSAWSAudience + `" + }, + { + "op": "add", + "path": "/spec/containers/0/command/1", + "value": "--service-account-issuer=` + accountIssuer + `" + }, + { + "op": "add", + "path": "/spec/containers/0/command/1", + "value": "--service-account-jwks-uri=` + accountIssuer + `/openid/v1/jwks" + } + ]` + + for i, _ := range controlPlanes.Items { + // files have to be unique so rebuild and toss the ones we're going to add + files := []v1beta12.File{} + for _, file := range controlPlanes.Items[i].Spec.KubeadmConfigSpec.Files { + if file.Path != "/etc/kubernetes/patches/kube-apiserver0+json.json" { + files = append(files, file) + } else if file.Content == patchContent { + return nil // nothing to reconcile + } + } + + controlPlanes.Items[i].Spec.KubeadmConfigSpec.Files = append(files, + // command starts with 0 == kube-apiserver, json patch add will insert at the position and shift the array + v1beta12.File{ + Path: "/etc/kubernetes/patches/kube-apiserver0+json.json", + Content: patchContent, + }) + + // panic checks to be safe + if controlPlanes.Items[i].Spec.KubeadmConfigSpec.InitConfiguration == nil { + controlPlanes.Items[i].Spec.KubeadmConfigSpec.InitConfiguration = &v1beta12.InitConfiguration{ + Patches: &v1beta12.Patches{}, + } + } + + if controlPlanes.Items[i].Spec.KubeadmConfigSpec.InitConfiguration.Patches == nil { + controlPlanes.Items[i].Spec.KubeadmConfigSpec.InitConfiguration.Patches = &v1beta12.Patches{} + } + + // set the patch directory for kubeadmn init to apply before booting apiserver + controlPlanes.Items[i].Spec.KubeadmConfigSpec.InitConfiguration.Patches.Directory = "/etc/kubernetes/patches" + + if err := managementClient.Update(ctx, &controlPlanes.Items[i]); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cloud/services/iam/podidentitywebhook.go b/pkg/cloud/services/iam/podidentitywebhook.go new file mode 100644 index 0000000000..7ba6f86247 --- /dev/null +++ b/pkg/cloud/services/iam/podidentitywebhook.go @@ -0,0 +1,340 @@ +package iam + +import ( + "context" + "errors" + + v14 "k8s.io/api/admissionregistration/v1" + v13 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + v12 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/cluster-api/api/v1beta1" +) + +const ( + podIdentityWebhookName = "pod-identity-webhook" + podIdentityWebhookImage = "amazon/amazon-eks-pod-identity-webhook:v0.4.0" +) + +func reconcileServiceAccount(ctx context.Context, ns string, remoteClient client.Client) error { + check := &corev1.ServiceAccount{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + Namespace: ns, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: objectMeta(podIdentityWebhookName, ns), + } + + return remoteClient.Create(ctx, sa) +} + +func reconcileClusterRole(ctx context.Context, ns string, remoteClient client.Client) error { + check := &v12.ClusterRole{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + Namespace: ns, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + + cr := &v12.ClusterRole{ + ObjectMeta: objectMeta(podIdentityWebhookName, ns), + Rules: []v12.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "update", "patch"}, + ResourceNames: []string{podIdentityWebhookName}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests"}, + Verbs: []string{"create", "get", "list", "watch"}, + }, + }, + } + + return remoteClient.Create(ctx, cr) +} + +func reconcileClusterRoleBinding(ctx context.Context, ns string, remoteClient client.Client) error { + check := &v12.ClusterRoleBinding{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + + crb := &v12.ClusterRoleBinding{ + ObjectMeta: objectMeta(podIdentityWebhookName, ""), + RoleRef: v12.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: podIdentityWebhookName, + }, + Subjects: []v12.Subject{ + { + Kind: "ServiceAccount", + Name: podIdentityWebhookName, + Namespace: ns, + }, + }, + } + + return remoteClient.Create(ctx, crb) +} + +func reconcileService(ctx context.Context, ns string, remoteClient client.Client) error { + check := &corev1.Service{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + Namespace: ns, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + service := &corev1.Service{ + ObjectMeta: objectMeta(podIdentityWebhookName, ns), + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: 443, + TargetPort: intstr.FromInt(443), + }, + }, + Selector: map[string]string{ + "app": podIdentityWebhookName, + }, + }, + } + + return remoteClient.Create(ctx, service) +} + +func reconcileDeployment(ctx context.Context, ns string, secret *corev1.Secret, remoteClient client.Client) error { + check := &v13.Deployment{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + Namespace: ns, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + + replicas := int32(1) + deployment := &v13.Deployment{ + ObjectMeta: objectMeta(podIdentityWebhookName, ns), + Spec: v13.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": podIdentityWebhookName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": podIdentityWebhookName, + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: podIdentityWebhookName, + Containers: []corev1.Container{ + { + Name: podIdentityWebhookName, + Image: podIdentityWebhookImage, + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "webhook-certs", + MountPath: "/etc/webhook/certs/", + ReadOnly: false, + }, + }, + Command: []string{ + "/webhook", + "--in-cluster=false", + "--namespace=" + ns, + "--service-name=" + podIdentityWebhookName, + "--annotation-prefix=eks.amazonaws.com", + "--token-audience=sts.amazonaws.com", + "--logtostderr", + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "webhook-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secret.Name, + }, + }, + }, + }, + }, + }, + }, + } + + return remoteClient.Create(ctx, deployment) +} + +func reconcileMutatingWebHook(ctx context.Context, ns string, secret *corev1.Secret, remoteClient client.Client) error { + check := &v14.MutatingWebhookConfiguration{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: podIdentityWebhookName, + Namespace: ns, + }, check); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if check.UID != "" { + return nil + } + + caBundle, ok := secret.Data["ca.crt"] + if !ok { + return errors.New("no CA certificate for the pod identity webhook certificate") + } + + mwhMeta := objectMeta(podIdentityWebhookName, ns) + fail := v14.Ignore + none := v14.SideEffectClassNone + mutate := "/mutate" + mwh := &v14.MutatingWebhookConfiguration{ + ObjectMeta: mwhMeta, + Webhooks: []v14.MutatingWebhook{ + { + Name: podIdentityWebhookName + ".amazonaws.com", + FailurePolicy: &fail, + ClientConfig: v14.WebhookClientConfig{ + Service: &v14.ServiceReference{ + Name: podIdentityWebhookName, + Namespace: ns, + Path: &mutate, + }, + CABundle: caBundle, + }, + Rules: []v14.RuleWithOperations{ + { + Operations: []v14.OperationType{v14.Create}, + Rule: v14.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + SideEffects: &none, + AdmissionReviewVersions: []string{"v1beta1"}, + }, + }, + } + + return remoteClient.Create(ctx, mwh) +} + +// reconcilePodIdentityWebhookComponents will create sa, cr, crb, service, deployment and a mutating webhook in kube-system. The +// only difference between this and upstream is we are using cert-manager in the management cluster to create a cert +// instead installing it in the work load cluster. +// https://github.com/aws/amazon-eks-pod-identity-webhook/tree/master/deploy +func reconcilePodIdentityWebhookComponents(ctx context.Context, ns string, secret *corev1.Secret, remoteClient client.Client) error { + // TODO only creates the object if they don't exist, could create some comparison logic for updates + if err := reconcileServiceAccount(ctx, ns, remoteClient); err != nil { + return err + } + + if err := reconcileClusterRole(ctx, ns, remoteClient); err != nil { + return err + } + + if err := reconcileClusterRoleBinding(ctx, ns, remoteClient); err != nil { + return err + } + + if err := reconcileService(ctx, ns, remoteClient); err != nil { + return err + } + + if err := reconcileDeployment(ctx, ns, secret, remoteClient); err != nil { + return err + } + + if err := reconcileMutatingWebHook(ctx, ns, secret, remoteClient); err != nil { + return err + } + + return nil +} + +func objectMeta(name, namespace string) metav1.ObjectMeta { + meta := metav1.ObjectMeta{ + Labels: map[string]string{ + v1beta1.ProviderLabelName: "infrastructure-aws", + }, + Name: name, + } + if namespace != "" { + meta.Namespace = namespace + } + return meta +} + +// reconcileCertifcateSecret takes a secret and moves it to the workload cluster. +func reconcileCertifcateSecret(ctx context.Context, cert *corev1.Secret, remoteClient client.Client) error { + // check if the secret was created by cert-manager + certCheck := &corev1.Secret{} + if err := remoteClient.Get(ctx, types.NamespacedName{ + Name: cert.Name, + Namespace: cert.Namespace, + }, certCheck); err != nil && !apierrors.IsNotFound(err) { + // will return not found if waiting for cert-manager and will reconcile again later due to error + return err + } + + if certCheck.UID == "" { + cert.ResourceVersion = "" + return remoteClient.Create(ctx, cert) + } + + return nil +} diff --git a/pkg/cloud/services/iam/reconcilers.go b/pkg/cloud/services/iam/reconcilers.go new file mode 100644 index 0000000000..af96d3b2c4 --- /dev/null +++ b/pkg/cloud/services/iam/reconcilers.go @@ -0,0 +1,179 @@ +package iam + +import ( + "context" + "fmt" + "path" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + v1certmanager "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/converters" + clusterapiv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// reconcilePodIdentityWebhook generates certs and starts the webhook in the workload cluster +// https://github.com/aws/amazon-eks-pod-identity-webhook +// 1. generate webhook certs via cert-manager in the management cluster +// 2. push cert secret down to the workload cluster +// 3. deploy pod identity webhook components with mounted certs (rbac,deployment,mwh,service). +func (s *Service) reconcilePodIdentityWebhook(ctx context.Context) error { + certName := fmt.Sprintf(PodIdentityWebhookCertificateFormat, s.scope.Name()) + certSecret, err := certificateSecret(ctx, + certName, s.scope.Namespace(), + fmt.Sprintf(SelfsignedIssuerFormat, s.scope.Name()), []string{ + fmt.Sprintf("%s.%s.svc", podIdentityWebhookName, s.scope.Namespace()), + fmt.Sprintf("%s.%s.svc.cluster.local", podIdentityWebhookName, s.scope.Namespace()), + }, s.scope.ManagementClient()) + + if err != nil { + return err + } + + remoteClient, err := s.scope.RemoteClient() + if err != nil { + return err + } + + // switch it to kube-system and move it to the remote cluster + if err := reconcileCertifcateSecret(ctx, certSecret, remoteClient); err != nil { + return err + } + + if err := reconcilePodIdentityWebhookComponents(ctx, s.scope.Namespace(), certSecret, remoteClient); err != nil { + return err + } + + return nil +} + +// reconcileSelfsignedIssuer create a selfsigned issuer at the cluster level. +func (s *Service) reconcileSelfsignedIssuer(ctx context.Context) error { + mgmtClient := s.scope.ManagementClient() + issuerName := fmt.Sprintf(SelfsignedIssuerFormat, s.scope.Name()) + issuer := &v1certmanager.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + clusterapiv1.ProviderLabelName: "infrastructure-aws", + }, + Name: issuerName, + Namespace: s.scope.Namespace(), + }, + Spec: v1certmanager.IssuerSpec{ + IssuerConfig: v1certmanager.IssuerConfig{ + SelfSigned: &v1certmanager.SelfSignedIssuer{}, + }, + }, + } + + if err := mgmtClient.Get(ctx, types.NamespacedName{ + Name: issuerName, + Namespace: s.scope.Namespace(), + }, issuer); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if issuer.UID != "" { + return nil + } + + return mgmtClient.Create(ctx, issuer) +} + +// CreateOIDCProvider will create an OIDC provider. +func (s *Service) reconcileIdentityProvider(_ context.Context) error { + s3Host := fmt.Sprintf(S3HostFormat, s.scope.Region()) + thumbprint, err := fetchRootCAThumbprint(s3Host, 443) + if err != nil { + return err + } + + oidcURL := "https://" + path.Join(s3Host, s.scope.Bucket().Name) + arn, err := findAndVerifyOIDCProvider(oidcURL, thumbprint, s.IAMClient) + if err != nil { + return err + } + + // find and verify confirms it's in IAM but if the status is not set we still want update + providerStatus := s.scope.OIDCProviderStatus() + if providerStatus.ARN != "" && providerStatus.ARN == arn { + return nil + } + + if arn == "" { + var tags []*iam.Tag + tags = append(tags, &iam.Tag{ + Key: aws.String(v1beta1.ClusterAWSCloudProviderTagKey(s.scope.Name())), + Value: aws.String(string(v1beta1.ResourceLifecycleOwned)), + }) + + input := iam.CreateOpenIDConnectProviderInput{ + ClientIDList: aws.StringSlice([]string{STSAWSAudience}), + ThumbprintList: aws.StringSlice([]string{thumbprint}), + Url: aws.String(oidcURL), + Tags: tags, + } + provider, err := s.IAMClient.CreateOpenIDConnectProvider(&input) + if err != nil { + return errors.Wrap(err, "error creating provider") + } + arn = aws.StringValue(provider.OpenIDConnectProviderArn) + } + + providerStatus.ARN = arn + oidcTrustPolicy := buildOIDCTrustPolicy(providerStatus.ARN) + policy, err := converters.IAMPolicyDocumentToJSON(oidcTrustPolicy) + if err != nil { + return errors.Wrap(err, "failed to parse IAM policy") + } + providerStatus.TrustPolicy = WhitespaceRegex.ReplaceAllString(policy, "") + return s.scope.PatchObject() +} + +// reconcileTrustPolicyConfigMap make sure the remote cluster has the config map of the trust policy, this enables +// the remote cluster to have everything it needs to create roles for services accounts. +func (s *Service) reconcileTrustPolicyConfigMap(ctx context.Context) error { + remoteClient, err := s.scope.RemoteClient() + if err != nil { + return err + } + + configMapRef := types.NamespacedName{ + Name: TrustPolicyConfigMapName, + Namespace: TrustPolicyConfigMapNamespace, + } + + trustPolicyConfigMap := &corev1.ConfigMap{} + err = remoteClient.Get(ctx, configMapRef, trustPolicyConfigMap) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("getting %s/%s config map: %w", TrustPolicyConfigMapNamespace, TrustPolicyConfigMapName, err) + } + + policy, err := converters.IAMPolicyDocumentToJSON(buildOIDCTrustPolicy(s.scope.OIDCProviderStatus().ARN)) + if err != nil { + return errors.Wrap(err, "failed to parse IAM policy") + } + + if tp, ok := trustPolicyConfigMap.Data[TrustPolicyJSON]; ok && tp == policy { + return nil // trust policy in the kube is the same as generated, don't update + } + + trustPolicyConfigMap.Data = map[string]string{ + TrustPolicyJSON: policy, + } + + if trustPolicyConfigMap.UID == "" { + trustPolicyConfigMap.Name = TrustPolicyConfigMapName + trustPolicyConfigMap.Namespace = TrustPolicyConfigMapNamespace + return remoteClient.Create(ctx, trustPolicyConfigMap) + } + + return remoteClient.Update(ctx, trustPolicyConfigMap) +} diff --git a/pkg/cloud/services/iamauth/reconcile.go b/pkg/cloud/services/iamauth/reconcile.go index 7e2cd3e37a..a71660ba74 100644 --- a/pkg/cloud/services/iamauth/reconcile.go +++ b/pkg/cloud/services/iamauth/reconcile.go @@ -19,6 +19,7 @@ package iamauth import ( "context" "fmt" + "sigs.k8s.io/cluster-api-provider-aws/util/system" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sts" @@ -48,7 +49,8 @@ func (s *Service) ReconcileIAMAuthenticator(ctx context.Context) error { return fmt.Errorf("getting aws-iam-authenticator backend: %w", err) } - roleARN := fmt.Sprintf("arn:aws:iam::%s:role/nodes%s", accountID, iamv1.DefaultNameSuffix) + partition := system.GetPartitionFromRegion(s.scope.Region()) + roleARN := fmt.Sprintf("arn:%s:iam::%s:role/nodes%s", partition, accountID, iamv1.DefaultNameSuffix) nodesRoleMapping := ekscontrolplanev1.RoleMapping{ RoleARN: roleARN, KubernetesMapping: ekscontrolplanev1.KubernetesMapping{ diff --git a/pkg/cloud/services/interfaces.go b/pkg/cloud/services/interfaces.go index 080acea34e..9f8869fade 100644 --- a/pkg/cloud/services/interfaces.go +++ b/pkg/cloud/services/interfaces.go @@ -17,6 +17,8 @@ limitations under the License. package services import ( + "context" + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" @@ -27,6 +29,8 @@ const ( TemporaryResourceID = "temporary-resource-id" // AnyIPv4CidrBlock is the CIDR block to match all IPv4 addresses. AnyIPv4CidrBlock = "0.0.0.0/0" + // AnyIPv6CidrBlock is the CIDR block to match all IPv6 addresses. + AnyIPv6CidrBlock = "::/0" ) // ASGInterface encapsulates the methods exposed to the machinepool @@ -59,14 +63,18 @@ type EC2Interface interface { TerminateInstanceAndWait(instanceID string) error DetachSecurityGroupsFromNetworkInterface(groups []string, interfaceID string) error - DiscoverLaunchTemplateAMI(scope *scope.MachinePoolScope) (*string, error) + ReconcileLaunchTemplate(scope scope.LaunchTemplateScope, canUpdateLaunchTemplate func() (bool, error), runPostLaunchTemplateUpdateOperation func() error) error + ReconcileTags(scope scope.LaunchTemplateScope, resourceServicesToUpdate []scope.ResourceServiceToUpdate) error + + DiscoverLaunchTemplateAMI(scope scope.LaunchTemplateScope) (*string, error) GetLaunchTemplate(id string) (lt *expinfrav1.AWSLaunchTemplate, userDataHash string, err error) GetLaunchTemplateID(id string) (string, error) - CreateLaunchTemplate(scope *scope.MachinePoolScope, imageID *string, userData []byte) (string, error) - CreateLaunchTemplateVersion(scope *scope.MachinePoolScope, imageID *string, userData []byte) error + GetLaunchTemplateLatestVersion(id string) (string, error) + CreateLaunchTemplate(scope scope.LaunchTemplateScope, imageID *string, userData []byte) (string, error) + CreateLaunchTemplateVersion(id string, scope scope.LaunchTemplateScope, imageID *string, userData []byte) error PruneLaunchTemplateVersions(id string) error DeleteLaunchTemplate(id string) error - LaunchTemplateNeedsUpdate(scope *scope.MachinePoolScope, incoming *expinfrav1.AWSLaunchTemplate, existing *expinfrav1.AWSLaunchTemplate) (bool, error) + LaunchTemplateNeedsUpdate(scope scope.LaunchTemplateScope, incoming *expinfrav1.AWSLaunchTemplate, existing *expinfrav1.AWSLaunchTemplate) (bool, error) DeleteBastion() error ReconcileBastion() error } @@ -96,6 +104,11 @@ type NetworkInterface interface { ReconcileNetwork() error } +type IAMInterface interface { + ReconcileOIDCProvider(ctx context.Context) error + DeleteOIDCProvider(ctx context.Context) error +} + // SecurityGroupInterface encapsulates the methods exposed to the cluster // controller. type SecurityGroupInterface interface { @@ -107,6 +120,7 @@ type SecurityGroupInterface interface { type ObjectStoreInterface interface { DeleteBucket() error ReconcileBucket() error - Delete(m *scope.MachineScope) error - Create(m *scope.MachineScope, data []byte) (objectURL string, err error) + Delete(key string) error + Create(key string, data []byte) (objectURL string, err error) + CreatePublic(key string, data []byte) (objectURL string, err error) } diff --git a/pkg/cloud/services/mock_services/ec2_interface_mock.go b/pkg/cloud/services/mock_services/ec2_interface_mock.go index a663f7b738..46ba71bc79 100644 --- a/pkg/cloud/services/mock_services/ec2_interface_mock.go +++ b/pkg/cloud/services/mock_services/ec2_interface_mock.go @@ -68,7 +68,7 @@ func (mr *MockEC2InterfaceMockRecorder) CreateInstance(arg0, arg1, arg2 interfac } // CreateLaunchTemplate mocks base method. -func (m *MockEC2Interface) CreateLaunchTemplate(arg0 *scope.MachinePoolScope, arg1 *string, arg2 []byte) (string, error) { +func (m *MockEC2Interface) CreateLaunchTemplate(arg0 scope.LaunchTemplateScope, arg1 *string, arg2 []byte) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateLaunchTemplate", arg0, arg1, arg2) ret0, _ := ret[0].(string) @@ -83,17 +83,17 @@ func (mr *MockEC2InterfaceMockRecorder) CreateLaunchTemplate(arg0, arg1, arg2 in } // CreateLaunchTemplateVersion mocks base method. -func (m *MockEC2Interface) CreateLaunchTemplateVersion(arg0 *scope.MachinePoolScope, arg1 *string, arg2 []byte) error { +func (m *MockEC2Interface) CreateLaunchTemplateVersion(arg0 string, arg1 scope.LaunchTemplateScope, arg2 *string, arg3 []byte) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateLaunchTemplateVersion", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreateLaunchTemplateVersion", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // CreateLaunchTemplateVersion indicates an expected call of CreateLaunchTemplateVersion. -func (mr *MockEC2InterfaceMockRecorder) CreateLaunchTemplateVersion(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockEC2InterfaceMockRecorder) CreateLaunchTemplateVersion(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLaunchTemplateVersion", reflect.TypeOf((*MockEC2Interface)(nil).CreateLaunchTemplateVersion), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLaunchTemplateVersion", reflect.TypeOf((*MockEC2Interface)(nil).CreateLaunchTemplateVersion), arg0, arg1, arg2, arg3) } // DeleteBastion mocks base method. @@ -139,7 +139,7 @@ func (mr *MockEC2InterfaceMockRecorder) DetachSecurityGroupsFromNetworkInterface } // DiscoverLaunchTemplateAMI mocks base method. -func (m *MockEC2Interface) DiscoverLaunchTemplateAMI(arg0 *scope.MachinePoolScope) (*string, error) { +func (m *MockEC2Interface) DiscoverLaunchTemplateAMI(arg0 scope.LaunchTemplateScope) (*string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DiscoverLaunchTemplateAMI", arg0) ret0, _ := ret[0].(*string) @@ -229,6 +229,21 @@ func (mr *MockEC2InterfaceMockRecorder) GetLaunchTemplateID(arg0 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLaunchTemplateID", reflect.TypeOf((*MockEC2Interface)(nil).GetLaunchTemplateID), arg0) } +// GetLaunchTemplateLatestVersion mocks base method. +func (m *MockEC2Interface) GetLaunchTemplateLatestVersion(arg0 string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLaunchTemplateLatestVersion", arg0) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLaunchTemplateLatestVersion indicates an expected call of GetLaunchTemplateLatestVersion. +func (mr *MockEC2InterfaceMockRecorder) GetLaunchTemplateLatestVersion(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLaunchTemplateLatestVersion", reflect.TypeOf((*MockEC2Interface)(nil).GetLaunchTemplateLatestVersion), arg0) +} + // GetRunningInstanceByTags mocks base method. func (m *MockEC2Interface) GetRunningInstanceByTags(arg0 *scope.MachineScope) (*v1beta1.Instance, error) { m.ctrl.T.Helper() @@ -260,7 +275,7 @@ func (mr *MockEC2InterfaceMockRecorder) InstanceIfExists(arg0 interface{}) *gomo } // LaunchTemplateNeedsUpdate mocks base method. -func (m *MockEC2Interface) LaunchTemplateNeedsUpdate(arg0 *scope.MachinePoolScope, arg1, arg2 *v1beta10.AWSLaunchTemplate) (bool, error) { +func (m *MockEC2Interface) LaunchTemplateNeedsUpdate(arg0 scope.LaunchTemplateScope, arg1, arg2 *v1beta10.AWSLaunchTemplate) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "LaunchTemplateNeedsUpdate", arg0, arg1, arg2) ret0, _ := ret[0].(bool) @@ -302,6 +317,34 @@ func (mr *MockEC2InterfaceMockRecorder) ReconcileBastion() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileBastion", reflect.TypeOf((*MockEC2Interface)(nil).ReconcileBastion)) } +// ReconcileLaunchTemplate mocks base method. +func (m *MockEC2Interface) ReconcileLaunchTemplate(arg0 scope.LaunchTemplateScope, arg1 func() (bool, error), arg2 func() error) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileLaunchTemplate", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconcileLaunchTemplate indicates an expected call of ReconcileLaunchTemplate. +func (mr *MockEC2InterfaceMockRecorder) ReconcileLaunchTemplate(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileLaunchTemplate", reflect.TypeOf((*MockEC2Interface)(nil).ReconcileLaunchTemplate), arg0, arg1, arg2) +} + +// ReconcileTags mocks base method. +func (m *MockEC2Interface) ReconcileTags(arg0 scope.LaunchTemplateScope, arg1 []scope.ResourceServiceToUpdate) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileTags", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconcileTags indicates an expected call of ReconcileTags. +func (mr *MockEC2InterfaceMockRecorder) ReconcileTags(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileTags", reflect.TypeOf((*MockEC2Interface)(nil).ReconcileTags), arg0, arg1) +} + // TerminateInstance mocks base method. func (m *MockEC2Interface) TerminateInstance(arg0 string) error { m.ctrl.T.Helper() diff --git a/pkg/cloud/services/mock_services/objectstore_machine_interface_mock.go b/pkg/cloud/services/mock_services/objectstore_machine_interface_mock.go index acb7cff238..191e7a32eb 100644 --- a/pkg/cloud/services/mock_services/objectstore_machine_interface_mock.go +++ b/pkg/cloud/services/mock_services/objectstore_machine_interface_mock.go @@ -24,7 +24,6 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" - scope "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" ) // MockObjectStoreInterface is a mock of ObjectStoreInterface interface. @@ -51,7 +50,7 @@ func (m *MockObjectStoreInterface) EXPECT() *MockObjectStoreInterfaceMockRecorde } // Create mocks base method. -func (m *MockObjectStoreInterface) Create(arg0 *scope.MachineScope, arg1 []byte) (string, error) { +func (m *MockObjectStoreInterface) Create(arg0 string, arg1 []byte) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Create", arg0, arg1) ret0, _ := ret[0].(string) @@ -65,8 +64,23 @@ func (mr *MockObjectStoreInterfaceMockRecorder) Create(arg0, arg1 interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockObjectStoreInterface)(nil).Create), arg0, arg1) } +// CreatePublic mocks base method. +func (m *MockObjectStoreInterface) CreatePublic(arg0 string, arg1 []byte) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreatePublic", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreatePublic indicates an expected call of CreatePublic. +func (mr *MockObjectStoreInterfaceMockRecorder) CreatePublic(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePublic", reflect.TypeOf((*MockObjectStoreInterface)(nil).CreatePublic), arg0, arg1) +} + // Delete mocks base method. -func (m *MockObjectStoreInterface) Delete(arg0 *scope.MachineScope) error { +func (m *MockObjectStoreInterface) Delete(arg0 string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Delete", arg0) ret0, _ := ret[0].(error) diff --git a/pkg/cloud/services/network/egress_only_gateways.go b/pkg/cloud/services/network/egress_only_gateways.go new file mode 100644 index 0000000000..73386b85cd --- /dev/null +++ b/pkg/cloud/services/network/egress_only_gateways.go @@ -0,0 +1,155 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/pkg/errors" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/converters" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/filter" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/wait" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/tags" + "sigs.k8s.io/cluster-api-provider-aws/pkg/record" + "sigs.k8s.io/cluster-api/util/conditions" +) + +func (s *Service) reconcileEgressOnlyInternetGateways() error { + if !s.scope.VPC().EnableIPv6 { + s.scope.V(4).Info("Skipping egress only internet gateways reconcile in not ipv6 mode") + return nil + } + + s.scope.V(2).Info("Reconciling egress only internet gateways") + + eigws, err := s.describeEgressOnlyVpcInternetGateways() + if awserrors.IsNotFound(err) { + if !s.scope.VPC().EnableIPv6 { + return errors.Errorf("failed to validate network: no egress only internet gateways found in VPC %q", s.scope.VPC().ID) + } + + ig, err := s.createEgressOnlyInternetGateway() + if err != nil { + return err + } + eigws = []*ec2.EgressOnlyInternetGateway{ig} + } else if err != nil { + return err + } + + gateway := eigws[0] + s.scope.VPC().EgressOnlyInternetGatewayID = gateway.EgressOnlyInternetGatewayId + + // Make sure tags are up to date. + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + buildParams := s.getEgressOnlyGatewayTagParams(*gateway.EgressOnlyInternetGatewayId) + tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client)) + if err := tagsBuilder.Ensure(converters.TagsToMap(gateway.Tags)); err != nil { + return false, err + } + return true, nil + }, awserrors.EgressOnlyInternetGatewayNotFound); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedTagEgressOnlyInternetGateway", "Failed to tag managed Egress Only Internet Gateway %q: %v", gateway.EgressOnlyInternetGatewayId, err) + return errors.Wrapf(err, "failed to tag egress only internet gateway %q", *gateway.EgressOnlyInternetGatewayId) + } + conditions.MarkTrue(s.scope.InfraCluster(), infrav1.EgressOnlyInternetGatewayReadyCondition) + return nil +} + +func (s *Service) deleteEgressOnlyInternetGateways() error { + if !s.scope.VPC().EnableIPv6 { + s.scope.V(4).Info("Skipping egress only internet gateway deletion in none ipv6 mode") + return nil + } + + eigws, err := s.describeEgressOnlyVpcInternetGateways() + if awserrors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + + for _, ig := range eigws { + deleteReq := &ec2.DeleteEgressOnlyInternetGatewayInput{ + EgressOnlyInternetGatewayId: ig.EgressOnlyInternetGatewayId, + } + + if _, err = s.EC2Client.DeleteEgressOnlyInternetGateway(deleteReq); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedDeleteEgressOnlyInternetGateway", "Failed to delete Egress Only Internet Gateway %q previously attached to VPC %q: %v", *ig.EgressOnlyInternetGatewayId, s.scope.VPC().ID, err) + return errors.Wrapf(err, "failed to delete egress only internet gateway %q", *ig.EgressOnlyInternetGatewayId) + } + + record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteEgressOnlyInternetGateway", "Deleted Egress Only Internet Gateway %q previously attached to VPC %q", *ig.EgressOnlyInternetGatewayId, s.scope.VPC().ID) + s.scope.Info("Deleted Egress Only Internet gateway in VPC", "egress-only-internet-gateway-id", *ig.EgressOnlyInternetGatewayId, "vpc-id", s.scope.VPC().ID) + } + + return nil +} + +func (s *Service) createEgressOnlyInternetGateway() (*ec2.EgressOnlyInternetGateway, error) { + ig, err := s.EC2Client.CreateEgressOnlyInternetGateway(&ec2.CreateEgressOnlyInternetGatewayInput{ + TagSpecifications: []*ec2.TagSpecification{ + tags.BuildParamsToTagSpecification(ec2.ResourceTypeEgressOnlyInternetGateway, s.getEgressOnlyGatewayTagParams(services.TemporaryResourceID)), + }, + VpcId: aws.String(s.scope.VPC().ID), + }) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedCreateEgressOnlyInternetGateway", "Failed to create new managed Egress Only Internet Gateway: %v", err) + return nil, errors.Wrap(err, "failed to create egress only internet gateway") + } + record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateEgressOnlyInternetGateway", "Created new managed Egree Only Internet Gateway %q", *ig.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId) + s.scope.Info("Created Egress Only Internet gateway", "egress-only-internet-gateway-id", *ig.EgressOnlyInternetGateway.EgressOnlyInternetGatewayId, "vpc-id", s.scope.VPC().ID) + + return ig.EgressOnlyInternetGateway, nil +} + +func (s *Service) describeEgressOnlyVpcInternetGateways() ([]*ec2.EgressOnlyInternetGateway, error) { + out, err := s.EC2Client.DescribeEgressOnlyInternetGateways(&ec2.DescribeEgressOnlyInternetGatewaysInput{ + Filters: []*ec2.Filter{ + filter.EC2.VPCAttachment(s.scope.VPC().ID), + }, + }) + if err != nil { + record.Eventf(s.scope.InfraCluster(), "FailedDescribeEgressOnlyInternetGateway", "Failed to describe egress only internet gateway in vpc %q: %v", s.scope.VPC().ID, err) + return nil, errors.Wrapf(err, "failed to describe egress only internet gateways in vpc %q", s.scope.VPC().ID) + } + + if len(out.EgressOnlyInternetGateways) == 0 { + return nil, awserrors.NewNotFound(fmt.Sprintf("no egress only internet gateways found in vpc %q", s.scope.VPC().ID)) + } + + return out.EgressOnlyInternetGateways, nil +} + +func (s *Service) getEgressOnlyGatewayTagParams(id string) infrav1.BuildParams { + name := fmt.Sprintf("%s-eigw", s.scope.Name()) + + return infrav1.BuildParams{ + ClusterName: s.scope.Name(), + ResourceID: id, + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: aws.String(name), + Role: aws.String(infrav1.CommonRoleTagValue), + Additional: s.scope.AdditionalTags(), + } +} diff --git a/pkg/cloud/services/network/egress_only_gateways_test.go b/pkg/cloud/services/network/egress_only_gateways_test.go new file mode 100644 index 0000000000..07e58e8f20 --- /dev/null +++ b/pkg/cloud/services/network/egress_only_gateways_test.go @@ -0,0 +1,266 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope" + "sigs.k8s.io/cluster-api-provider-aws/test/mocks" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +func TestReconcileEgressOnlyInternetGateways(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + testCases := []struct { + name string + input *infrav1.NetworkSpec + expect func(m *mocks.MockEC2APIMockRecorder) + }{ + { + name: "has eigw", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpc-egress-only-gateways", + EnableIPv6: true, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + }, + }, + expect: func(m *mocks.MockEC2APIMockRecorder) { + m.DescribeEgressOnlyInternetGateways(gomock.AssignableToTypeOf(&ec2.DescribeEgressOnlyInternetGatewaysInput{})). + Return(&ec2.DescribeEgressOnlyInternetGatewaysOutput{ + EgressOnlyInternetGateways: []*ec2.EgressOnlyInternetGateway{ + { + EgressOnlyInternetGatewayId: aws.String("eigw-0"), + Attachments: []*ec2.InternetGatewayAttachment{ + { + State: aws.String(ec2.AttachmentStatusAttached), + VpcId: aws.String("vpc-egress-only-gateways"), + }, + }, + }, + }, + }, nil) + + m.CreateTags(gomock.AssignableToTypeOf(&ec2.CreateTagsInput{})). + Return(nil, nil) + }, + }, + { + name: "no eigw attached, creates one", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + ID: "vpc-egress-only-gateways", + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + }, + }, + expect: func(m *mocks.MockEC2APIMockRecorder) { + m.DescribeEgressOnlyInternetGateways(gomock.AssignableToTypeOf(&ec2.DescribeEgressOnlyInternetGatewaysInput{})). + Return(&ec2.DescribeEgressOnlyInternetGatewaysOutput{}, nil) + + m.CreateEgressOnlyInternetGateway(gomock.AssignableToTypeOf(&ec2.CreateEgressOnlyInternetGatewayInput{})). + Return(&ec2.CreateEgressOnlyInternetGatewayOutput{ + EgressOnlyInternetGateway: &ec2.EgressOnlyInternetGateway{ + EgressOnlyInternetGatewayId: aws.String("igw-1"), + Tags: []*ec2.Tag{ + { + Key: aws.String(infrav1.ClusterTagKey("test-cluster")), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-eigw"), + }, + }, + Attachments: []*ec2.InternetGatewayAttachment{ + { + State: aws.String(ec2.AttachmentStatusAttached), + VpcId: aws.String("vpc-egress-only-gateways"), + }, + }, + }, + }, nil) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ec2Mock := mocks.NewMockEC2API(mockCtrl) + + scheme := runtime.NewScheme() + _ = infrav1.AddToScheme(scheme) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + scope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, + }, + AWSCluster: &infrav1.AWSCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: infrav1.AWSClusterSpec{ + NetworkSpec: *tc.input, + }, + }, + }) + if err != nil { + t.Fatalf("Failed to create test context: %v", err) + } + + tc.expect(ec2Mock.EXPECT()) + + s := NewService(scope) + s.EC2Client = ec2Mock + + if err := s.reconcileEgressOnlyInternetGateways(); err != nil { + t.Fatalf("got an unexpected error: %v", err) + } + }) + } +} + +func TestDeleteEgressOnlyInternetGateways(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + testCases := []struct { + name string + input *infrav1.NetworkSpec + expect func(m *mocks.MockEC2APIMockRecorder) + wantErr bool + }{ + { + name: "Should ignore deletion if vpc is not ipv6", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpc-gateways", + }, + }, + expect: func(m *mocks.MockEC2APIMockRecorder) {}, + }, + { + name: "Should ignore deletion if egress only internet gateway is not found", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + EnableIPv6: true, + ID: "vpc-gateways", + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + }, + }, + expect: func(m *mocks.MockEC2APIMockRecorder) { + m.DescribeEgressOnlyInternetGateways(gomock.Eq(&ec2.DescribeEgressOnlyInternetGatewaysInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("attachment.vpc-id"), + Values: aws.StringSlice([]string{"vpc-gateways"}), + }, + }, + })).Return(&ec2.DescribeEgressOnlyInternetGatewaysOutput{}, nil) + }, + }, + { + name: "Should successfully delete the egress only internet gateway", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpc-gateways", + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + EnableIPv6: true, + }, + }, + expect: func(m *mocks.MockEC2APIMockRecorder) { + m.DescribeEgressOnlyInternetGateways(gomock.AssignableToTypeOf(&ec2.DescribeEgressOnlyInternetGatewaysInput{})). + Return(&ec2.DescribeEgressOnlyInternetGatewaysOutput{ + EgressOnlyInternetGateways: []*ec2.EgressOnlyInternetGateway{ + { + EgressOnlyInternetGatewayId: aws.String("eigw-0"), + Attachments: []*ec2.InternetGatewayAttachment{ + { + State: aws.String(ec2.AttachmentStatusAttached), + VpcId: aws.String("vpc-gateways"), + }, + }, + }, + }, + }, nil) + m.DeleteEgressOnlyInternetGateway(&ec2.DeleteEgressOnlyInternetGatewayInput{ + EgressOnlyInternetGatewayId: aws.String("eigw-0"), + }).Return(&ec2.DeleteEgressOnlyInternetGatewayOutput{}, nil) + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ec2Mock := mocks.NewMockEC2API(mockCtrl) + + scheme := runtime.NewScheme() + err := infrav1.AddToScheme(scheme) + g.Expect(err).NotTo(HaveOccurred()) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + + scope, err := scope.NewClusterScope(scope.ClusterScopeParams{ + Client: client, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"}, + }, + AWSCluster: &infrav1.AWSCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: infrav1.AWSClusterSpec{ + NetworkSpec: *tc.input, + }, + }, + }) + g.Expect(err).NotTo(HaveOccurred()) + + tc.expect(ec2Mock.EXPECT()) + + s := NewService(scope) + s.EC2Client = ec2Mock + + err = s.deleteEgressOnlyInternetGateways() + if tc.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).NotTo(HaveOccurred()) + }) + } +} diff --git a/pkg/cloud/services/network/network.go b/pkg/cloud/services/network/network.go index d1639fb63b..2b9180c596 100644 --- a/pkg/cloud/services/network/network.go +++ b/pkg/cloud/services/network/network.go @@ -53,6 +53,12 @@ func (s *Service) ReconcileNetwork() (err error) { return err } + // Egress Only Internet Gateways. + if err := s.reconcileEgressOnlyInternetGateways(); err != nil { + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.EgressOnlyInternetGatewayReadyCondition, infrav1.EgressOnlyInternetGatewayFailedReason, infrautilconditions.ErrorConditionAfterInit(s.scope.ClusterObj()), err.Error()) + return err + } + // NAT Gateways. if err := s.reconcileNatGateways(); err != nil { conditions.MarkFalse(s.scope.InfraCluster(), infrav1.NatGatewaysReadyCondition, infrav1.NatGatewaysReconciliationFailedReason, infrautilconditions.ErrorConditionAfterInit(s.scope.ClusterObj()), err.Error()) @@ -132,6 +138,18 @@ func (s *Service) DeleteNetwork() (err error) { } conditions.MarkFalse(s.scope.InfraCluster(), infrav1.InternetGatewayReadyCondition, clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") + // Egress Only Internet Gateways. + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.EgressOnlyInternetGatewayReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") + if err := s.scope.PatchObject(); err != nil { + return err + } + + if err := s.deleteEgressOnlyInternetGateways(); err != nil { + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.EgressOnlyInternetGatewayReadyCondition, "DeletingFailed", clusterv1.ConditionSeverityWarning, err.Error()) + return err + } + conditions.MarkFalse(s.scope.InfraCluster(), infrav1.EgressOnlyInternetGatewayReadyCondition, clusterv1.DeletedReason, clusterv1.ConditionSeverityInfo, "") + // Subnets. conditions.MarkFalse(s.scope.InfraCluster(), infrav1.SubnetsReadyCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "") if err := s.scope.PatchObject(); err != nil { diff --git a/pkg/cloud/services/network/routetables.go b/pkg/cloud/services/network/routetables.go index 199328eb0a..8f4ee9427e 100644 --- a/pkg/cloud/services/network/routetables.go +++ b/pkg/cloud/services/network/routetables.go @@ -61,12 +61,18 @@ func (s *Service) reconcileRouteTables() error { return errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID) } routes = append(routes, s.getGatewayPublicRoute()) + if sn.IsIPv6 { + routes = append(routes, s.getGatewayPublicIPv6Route()) + } } else { natGatewayID, err := s.getNatGatewayForSubnet(&sn) if err != nil { return err } routes = append(routes, s.getNatGatewayPrivateRoute(natGatewayID)) + if sn.IsIPv6 { + routes = append(routes, s.getEgressOnlyInternetGateway()) + } } if rt, ok := subnetRouteMap[sn.ID]; ok { @@ -80,30 +86,13 @@ func (s *Service) reconcileRouteTables() error { for i := range routes { // Routes destination cidr blocks must be unique within a routing table. // If there is a mistmatch, we replace the routing association. - specRoute := routes[i] - if (currentRoute.DestinationCidrBlock != nil && // Manually-created routes can have .DestinationIpv6CidrBlock or .DestinationPrefixListId set instead. - *currentRoute.DestinationCidrBlock == *specRoute.DestinationCidrBlock) && - ((currentRoute.GatewayId != nil && *currentRoute.GatewayId != *specRoute.GatewayId) || - (currentRoute.NatGatewayId != nil && *currentRoute.NatGatewayId != *specRoute.NatGatewayId)) { - if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - if _, err := s.EC2Client.ReplaceRoute(&ec2.ReplaceRouteInput{ - RouteTableId: rt.RouteTableId, - DestinationCidrBlock: specRoute.DestinationCidrBlock, - GatewayId: specRoute.GatewayId, - NatGatewayId: specRoute.NatGatewayId, - }); err != nil { - return false, err - } - return true, nil - }); err != nil { - record.Warnf(s.scope.InfraCluster(), "FailedReplaceRoute", "Failed to replace outdated route on managed RouteTable %q: %v", *rt.RouteTableId, err) - return errors.Wrapf(err, "failed to replace outdated route on route table %q", *rt.RouteTableId) - } + if err := s.fixMismatchedRouting(routes[i], currentRoute, rt); err != nil { + return err } } } - // Make sure tags are up to date. + // Make sure tags are up-to-date. if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { buildParams := s.getRouteTableTagParams(*rt.RouteTableId, sn.IsPublic, sn.AvailabilityZone) tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client)) @@ -144,6 +133,50 @@ func (s *Service) reconcileRouteTables() error { return nil } +func (s *Service) fixMismatchedRouting(specRoute *ec2.Route, currentRoute *ec2.Route, rt *ec2.RouteTable) error { + var input *ec2.ReplaceRouteInput + if specRoute.DestinationCidrBlock != nil { + if (currentRoute.DestinationCidrBlock != nil && + *currentRoute.DestinationCidrBlock == *specRoute.DestinationCidrBlock) && + ((currentRoute.GatewayId != nil && *currentRoute.GatewayId != *specRoute.GatewayId) || + (currentRoute.NatGatewayId != nil && *currentRoute.NatGatewayId != *specRoute.NatGatewayId)) { + input = &ec2.ReplaceRouteInput{ + RouteTableId: rt.RouteTableId, + DestinationCidrBlock: specRoute.DestinationCidrBlock, + GatewayId: specRoute.GatewayId, + NatGatewayId: specRoute.NatGatewayId, + } + } + } + if specRoute.DestinationIpv6CidrBlock != nil { + if (currentRoute.DestinationIpv6CidrBlock != nil && + *currentRoute.DestinationIpv6CidrBlock == *specRoute.DestinationIpv6CidrBlock) && + ((currentRoute.GatewayId != nil && *currentRoute.GatewayId != *specRoute.GatewayId) || + (currentRoute.NatGatewayId != nil && *currentRoute.NatGatewayId != *specRoute.NatGatewayId)) { + input = &ec2.ReplaceRouteInput{ + RouteTableId: rt.RouteTableId, + DestinationIpv6CidrBlock: specRoute.DestinationIpv6CidrBlock, + DestinationPrefixListId: specRoute.DestinationPrefixListId, + GatewayId: specRoute.GatewayId, + NatGatewayId: specRoute.NatGatewayId, + EgressOnlyInternetGatewayId: specRoute.EgressOnlyInternetGatewayId, + } + } + } + if input != nil { + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + if _, err := s.EC2Client.ReplaceRoute(input); err != nil { + return false, err + } + return true, nil + }); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedReplaceRoute", "Failed to replace outdated route on managed RouteTable %q: %v", *rt.RouteTableId, err) + return errors.Wrapf(err, "failed to replace outdated route on route table %q", *rt.RouteTableId) + } + } + return nil +} + func (s *Service) describeVpcRouteTablesBySubnet() (map[string]*ec2.RouteTable, error) { rts, err := s.describeVpcRouteTables() if err != nil { @@ -286,8 +319,15 @@ func (s *Service) associateRouteTable(rt *infrav1.RouteTable, subnetID string) e func (s *Service) getNatGatewayPrivateRoute(natGatewayID string) *ec2.Route { return &ec2.Route{ - DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), NatGatewayId: aws.String(natGatewayID), + DestinationCidrBlock: aws.String(services.AnyIPv4CidrBlock), + } +} + +func (s *Service) getEgressOnlyInternetGateway() *ec2.Route { + return &ec2.Route{ + DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), + EgressOnlyInternetGatewayId: s.scope.VPC().EgressOnlyInternetGatewayID, } } @@ -298,6 +338,13 @@ func (s *Service) getGatewayPublicRoute() *ec2.Route { } } +func (s *Service) getGatewayPublicIPv6Route() *ec2.Route { + return &ec2.Route{ + DestinationIpv6CidrBlock: aws.String(services.AnyIPv6CidrBlock), + GatewayId: aws.String(*s.scope.VPC().InternetGatewayID), + } +} + func (s *Service) getRouteTableTagParams(id string, public bool, zone string) infrav1.BuildParams { var name strings.Builder diff --git a/pkg/cloud/services/network/routetables_test.go b/pkg/cloud/services/network/routetables_test.go index a1414b8e57..972fec2b7d 100644 --- a/pkg/cloud/services/network/routetables_test.go +++ b/pkg/cloud/services/network/routetables_test.go @@ -110,6 +110,176 @@ func TestReconcileRouteTables(t *testing.T) { After(publicRouteTable) }, }, + { + name: "no routes existing, single private and single public IPv6 enabled subnets, same AZ", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpc-routetables", + InternetGatewayID: aws.String("igw-01"), + EgressOnlyInternetGatewayID: aws.String("eigw-01"), + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234::/56", + IPv6Pool: "my-pool", + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + }, + Subnets: infrav1.Subnets{ + infrav1.SubnetSpec{ + ID: "subnet-routetables-private", + IsPublic: false, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1::/64", + AvailabilityZone: "us-east-1a", + }, + infrav1.SubnetSpec{ + ID: "subnet-routetables-public", + IsPublic: true, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:2::/64", + NatGatewayID: aws.String("nat-01"), + AvailabilityZone: "us-east-1a", + }, + }, + }, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeRouteTables(gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + privateRouteTable := m.CreateRouteTable(matchRouteTableInput(&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-routetables")})). + Return(&ec2.CreateRouteTableOutput{RouteTable: &ec2.RouteTable{RouteTableId: aws.String("rt-1")}}, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + NatGatewayId: aws.String("nat-01"), + DestinationCidrBlock: aws.String("0.0.0.0/0"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("::/0"), + EgressOnlyInternetGatewayId: aws.String("eigw-01"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rt-1"), + SubnetId: aws.String("subnet-routetables-private"), + })). + Return(&ec2.AssociateRouteTableOutput{}, nil). + After(privateRouteTable) + + publicRouteTable := m.CreateRouteTable(matchRouteTableInput(&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-routetables")})). + Return(&ec2.CreateRouteTableOutput{RouteTable: &ec2.RouteTable{RouteTableId: aws.String("rt-2")}}, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + GatewayId: aws.String("igw-01"), + DestinationCidrBlock: aws.String("0.0.0.0/0"), + RouteTableId: aws.String("rt-2"), + })). + After(publicRouteTable) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("::/0"), + GatewayId: aws.String("igw-01"), + RouteTableId: aws.String("rt-2"), + })). + After(publicRouteTable) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rt-2"), + SubnetId: aws.String("subnet-routetables-public"), + })). + Return(&ec2.AssociateRouteTableOutput{}, nil). + After(publicRouteTable) + }, + }, + { + name: "no routes existing, single private and single public IPv6 enabled subnets with existing Egress only IWG, same AZ", + input: &infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: "vpc-routetables", + InternetGatewayID: aws.String("igw-01"), + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234::/56", + IPv6Pool: "my-pool", + EgressOnlyInternetGatewayID: aws.String("eigw-01"), + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + }, + Subnets: infrav1.Subnets{ + infrav1.SubnetSpec{ + ID: "subnet-routetables-private", + IsPublic: false, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1::/64", + AvailabilityZone: "us-east-1a", + }, + infrav1.SubnetSpec{ + ID: "subnet-routetables-public", + IsPublic: true, + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:2::/64", + NatGatewayID: aws.String("nat-01"), + AvailabilityZone: "us-east-1a", + }, + }, + }, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeRouteTables(gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + privateRouteTable := m.CreateRouteTable(matchRouteTableInput(&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-routetables")})). + Return(&ec2.CreateRouteTableOutput{RouteTable: &ec2.RouteTable{RouteTableId: aws.String("rt-1")}}, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + NatGatewayId: aws.String("nat-01"), + DestinationCidrBlock: aws.String("0.0.0.0/0"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("::/0"), + EgressOnlyInternetGatewayId: aws.String("eigw-01"), + RouteTableId: aws.String("rt-1"), + })). + After(privateRouteTable) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rt-1"), + SubnetId: aws.String("subnet-routetables-private"), + })). + Return(&ec2.AssociateRouteTableOutput{}, nil). + After(privateRouteTable) + + publicRouteTable := m.CreateRouteTable(matchRouteTableInput(&ec2.CreateRouteTableInput{VpcId: aws.String("vpc-routetables")})). + Return(&ec2.CreateRouteTableOutput{RouteTable: &ec2.RouteTable{RouteTableId: aws.String("rt-2")}}, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + GatewayId: aws.String("igw-01"), + DestinationCidrBlock: aws.String("0.0.0.0/0"), + RouteTableId: aws.String("rt-2"), + })). + After(publicRouteTable) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationIpv6CidrBlock: aws.String("::/0"), + GatewayId: aws.String("igw-01"), + RouteTableId: aws.String("rt-2"), + })). + After(publicRouteTable) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rt-2"), + SubnetId: aws.String("subnet-routetables-public"), + })). + Return(&ec2.AssociateRouteTableOutput{}, nil). + After(publicRouteTable) + }, + }, { name: "subnets in different availability zones, returns error", input: &infrav1.NetworkSpec{ diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index 3355e56d06..b678749f5a 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -19,6 +19,7 @@ package network import ( "fmt" "math/rand" + "net" "sort" "strings" @@ -118,6 +119,7 @@ func (s *Service) reconcileSubnets() error { if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.ID, existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags) tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client)) + s.removeNonClusterSharedTags(subnetTags, existingSubnet.Tags) if err := tagsBuilder.Ensure(existingSubnet.Tags); err != nil { return false, err } @@ -212,38 +214,108 @@ func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { } // 1 private subnet for each AZ plus 1 other subnet that will be further sub-divided for the public subnets + // All subnets will have an ipv4 address for now as well. We aren't supporting ipv6-only yet. numSubnets := len(zones) + 1 - subnetCIDRs, err := cidr.SplitIntoSubnetsIPv4(s.scope.VPC().CidrBlock, numSubnets) + var ( + subnetCIDRs []*net.IPNet + publicSubnetCIDRs []*net.IPNet + ipv6SubnetCIDRs []*net.IPNet + publicIPv6SubnetCIDRs []*net.IPNet + privateIPv6SubnetCIDRs []*net.IPNet + ) + subnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(s.scope.VPC().CidrBlock, numSubnets) if err != nil { - return nil, errors.Wrapf(err, "failed splitting VPC CIDR %s into subnets", s.scope.VPC().CidrBlock) + return nil, errors.Wrapf(err, "failed splitting VPC CIDR %q into subnets", s.scope.VPC().CidrBlock) } - publicSubnetCIDRs, err := cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) + publicSubnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) if err != nil { - return nil, errors.Wrapf(err, "failed splitting CIDR %s into public subnets", subnetCIDRs[0].String()) + return nil, errors.Wrapf(err, "failed splitting CIDR %q into public subnets", subnetCIDRs[0].String()) } privateSubnetCIDRs := append(subnetCIDRs[:0], subnetCIDRs[1:]...) + if s.scope.VPC().EnableIPv6 { + ipv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(s.scope.VPC().IPv6CidrBlock, numSubnets) + if err != nil { + return nil, errors.Wrapf(err, "failed splitting IPv6 VPC CIDR %q into subnets", s.scope.VPC().IPv6CidrBlock) + } + + // We need to take the last, so it doesn't conflict with the rest. The subnetID is increment each time by 1. + publicIPv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String(), len(zones)) + if err != nil { + return nil, errors.Wrapf(err, "failed splitting IPv6 CIDR %q into public subnets", ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String()) + } + // TODO: this might need to be the last instead of the first.. + privateIPv6SubnetCIDRs = append(ipv6SubnetCIDRs[:0], ipv6SubnetCIDRs[1:]...) + } + subnets := infrav1.Subnets{} for i, zone := range zones { - subnets = append(subnets, infrav1.SubnetSpec{ + publicSubnet := infrav1.SubnetSpec{ CidrBlock: publicSubnetCIDRs[i].String(), AvailabilityZone: zone, IsPublic: true, - }) - subnets = append(subnets, infrav1.SubnetSpec{ + } + privateSubnet := infrav1.SubnetSpec{ CidrBlock: privateSubnetCIDRs[i].String(), AvailabilityZone: zone, IsPublic: false, - }) + } + + if s.scope.VPC().EnableIPv6 { + publicSubnet.IPv6CidrBlock = publicIPv6SubnetCIDRs[i].String() + publicSubnet.IsIPv6 = true + privateSubnet.IPv6CidrBlock = privateIPv6SubnetCIDRs[i].String() + privateSubnet.IsIPv6 = true + } + + subnets = append(subnets, publicSubnet, privateSubnet) } return subnets, nil } func (s *Service) deleteSubnets() error { + s.scope.V(0).Info("Deleting subnets") if s.scope.VPC().IsUnmanaged(s.scope.Name()) { - s.scope.V(4).Info("Skipping subnets deletion in unmanaged mode") + + s.scope.V(0).Info("Deleting subnet tags if vpc is unmanaged") + + // For deletion of the subnet tag while the static placement cluster is getting deleted: + // for all the subnets, the specific tag of key and value is fetched (in this case it will be equal to one's of the cluster getting deleted) + // If the tag is found, it will be deleted. If not found, an error will be returned. + // The fetching of tags is done in the same way tags are described using AWS CLI. + existing, err := s.describeSubnets() + if err != nil { + return err + } + + for i := range existing.Subnets { + + describeTagsInput := &ec2.DescribeTagsInput{Filters: []*ec2.Filter{ + {Name: aws.String("resource-type"), Values: []*string{aws.String("subnet")}}, + {Name: aws.String("key"), Values: []*string{aws.String(infrav1.NameKubernetesAWSCloudProviderPrefix + s.scope.KubernetesClusterName())}}, + {Name: aws.String("value"), Values: []*string{aws.String(string(infrav1.ResourceLifecycleShared))}}, + }, + } + + if fetchedTags, err := s.EC2Client.DescribeTags(describeTagsInput); err != nil { + return errors.Wrapf(err, "failed to fetch tags for resource %q", *existing.Subnets[i].SubnetId) + } else if len(fetchedTags.Tags) > 0 { + s.scope.V(0).Info("Found a tag for deletion") + // Create the DeleteTags input + deleteTagsInput := &ec2.DeleteTagsInput{ + Resources: []*string{existing.Subnets[i].SubnetId}, + Tags: []*ec2.Tag{{Key: aws.String(infrav1.NameKubernetesAWSCloudProviderPrefix + s.scope.KubernetesClusterName()), Value: aws.String(string(infrav1.ResourceLifecycleShared))}}, + } + + // Delete tags in AWS. + if _, err = s.EC2Client.DeleteTags(deleteTagsInput); err != nil { + return errors.Wrapf(err, "failed to delete tags for resource %q", *existing.Subnets[i].SubnetId) + } + } + } + s.scope.V(0).Info("Skipping subnets deletion in unmanaged mode") return nil } @@ -284,11 +356,17 @@ func (s *Service) describeVpcSubnets() (infrav1.Subnets, error) { for _, ec2sn := range sns.Subnets { spec := infrav1.SubnetSpec{ ID: *ec2sn.SubnetId, - CidrBlock: *ec2sn.CidrBlock, AvailabilityZone: *ec2sn.AvailabilityZone, Tags: converters.TagsToMap(ec2sn.Tags), } - + // For IPv6 subnets, both, ipv4 and 6 have to be defined so pods can have ipv6 cidr ranges. + spec.CidrBlock = aws.StringValue(ec2sn.CidrBlock) + for _, set := range ec2sn.Ipv6CidrBlockAssociationSet { + if *set.Ipv6CidrBlockState.State == ec2.SubnetCidrBlockStateCodeAssociated { + spec.IPv6CidrBlock = aws.StringValue(set.Ipv6CidrBlock) + spec.IsIPv6 = true + } + } // A subnet is public if it's tagged as such... if spec.Tags.GetRole() == infrav1.PublicRoleTagValue { spec.IsPublic = true @@ -341,7 +419,7 @@ func (s *Service) describeSubnets() (*ec2.DescribeSubnetsOutput, error) { } func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, error) { - out, err := s.EC2Client.CreateSubnet(&ec2.CreateSubnetInput{ + input := &ec2.CreateSubnetInput{ VpcId: aws.String(s.scope.VPC().ID), CidrBlock: aws.String(sn.CidrBlock), AvailabilityZone: aws.String(sn.AvailabilityZone), @@ -351,52 +429,86 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags), ), }, - }) + } + if s.scope.VPC().EnableIPv6 { + input.Ipv6CidrBlock = aws.String(sn.IPv6CidrBlock) + sn.IsIPv6 = true + } + out, err := s.EC2Client.CreateSubnet(input) if err != nil { record.Warnf(s.scope.InfraCluster(), "FailedCreateSubnet", "Failed creating new managed Subnet %v", err) return nil, errors.Wrap(err, "failed to create subnet") } record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateSubnet", "Created new managed Subnet %q", *out.Subnet.SubnetId) - s.scope.Info("Created subnet", "id", *out.Subnet.SubnetId, "public", sn.IsPublic, "az", sn.AvailabilityZone, "cidr", sn.CidrBlock) + s.scope.Info("Created subnet", "id", *out.Subnet.SubnetId, "public", sn.IsPublic, "az", sn.AvailabilityZone, "cidr", sn.CidrBlock, "ipv6", sn.IsIPv6, "ipv6-cidr", sn.IPv6CidrBlock) wReq := &ec2.DescribeSubnetsInput{SubnetIds: []*string{out.Subnet.SubnetId}} if err := s.EC2Client.WaitUntilSubnetAvailable(wReq); err != nil { return nil, errors.Wrapf(err, "failed to wait for subnet %q", *out.Subnet.SubnetId) } - if sn.IsPublic { - attReq := &ec2.ModifySubnetAttributeInput{ - MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ - Value: aws.Bool(true), - }, - SubnetId: out.Subnet.SubnetId, + // This has to be done separately, because: + // InvalidParameterCombination: Only one subnet attribute can be modified at a time + if sn.IsIPv6 { + // regardless of the subnet being public or not, ipv6 address needs to be assigned + // on creation. There is no such thing as private ipv6 address. + if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { + if _, err := s.EC2Client.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + SubnetId: out.Subnet.SubnetId, + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }); err != nil { + return false, err + } + return true, nil + }, awserrors.SubnetNotFound); err != nil { + record.Warnf(s.scope.InfraCluster(), "FailedModifySubnetAttributes", "Failed modifying managed Subnet %q attributes: %v", *out.Subnet.SubnetId, err) + return nil, errors.Wrapf(err, "failed to set subnet %q attribute assign ipv6 address on creation", *out.Subnet.SubnetId) } + record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) + } + if sn.IsPublic { if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) { - if _, err := s.EC2Client.ModifySubnetAttribute(attReq); err != nil { + if _, err := s.EC2Client.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + SubnetId: out.Subnet.SubnetId, + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + }); err != nil { return false, err } return true, nil }, awserrors.SubnetNotFound); err != nil { record.Warnf(s.scope.InfraCluster(), "FailedModifySubnetAttributes", "Failed modifying managed Subnet %q attributes: %v", *out.Subnet.SubnetId, err) - return nil, errors.Wrapf(err, "failed to set subnet %q attributes", *out.Subnet.SubnetId) + return nil, errors.Wrapf(err, "failed to set subnet %q attribute assign ipv4 address on creation", *out.Subnet.SubnetId) } record.Eventf(s.scope.InfraCluster(), "SuccessfulModifySubnetAttributes", "Modified managed Subnet %q attributes", *out.Subnet.SubnetId) } + subnet := &infrav1.SubnetSpec{ + ID: *out.Subnet.SubnetId, + AvailabilityZone: *out.Subnet.AvailabilityZone, + CidrBlock: *out.Subnet.CidrBlock, // TODO: this will panic in case of IPv6 only subnets... + IsPublic: sn.IsPublic, + } + for _, set := range out.Subnet.Ipv6CidrBlockAssociationSet { + if *set.Ipv6CidrBlockState.State == ec2.SubnetCidrBlockStateCodeAssociated { + subnet.IPv6CidrBlock = aws.StringValue(set.Ipv6CidrBlock) + subnet.IsIPv6 = true + } + } + s.scope.V(2).Info("Created new subnet in VPC with cidr and availability zone ", "subnet-id", *out.Subnet.SubnetId, "vpc-id", *out.Subnet.VpcId, "cidr-block", *out.Subnet.CidrBlock, + "ipv6-cidr-block", subnet.IPv6CidrBlock, "availability-zone", *out.Subnet.AvailabilityZone) - return &infrav1.SubnetSpec{ - ID: *out.Subnet.SubnetId, - AvailabilityZone: *out.Subnet.AvailabilityZone, - CidrBlock: *out.Subnet.CidrBlock, - IsPublic: sn.IsPublic, - }, nil + return subnet, nil } func (s *Service) deleteSubnet(id string) error { @@ -455,3 +567,19 @@ func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, } } } + +func (s *Service) removeNonClusterSharedTags(clusterSubnetTags, VPCSubnetTags infrav1.Tags) { + + for key, value := range clusterSubnetTags { + if key != infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName()) && value == string(infrav1.ResourceLifecycleShared) { + delete(clusterSubnetTags, key) + } + } + + for key, value := range VPCSubnetTags { + if key != infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName()) && value == string(infrav1.ResourceLifecycleShared) { + delete(VPCSubnetTags, key) + } + } + +} diff --git a/pkg/cloud/services/network/subnets_test.go b/pkg/cloud/services/network/subnets_test.go index c1ae1127c6..f8d120f379 100644 --- a/pkg/cloud/services/network/subnets_test.go +++ b/pkg/cloud/services/network/subnets_test.go @@ -160,6 +160,143 @@ func TestReconcileSubnets(t *testing.T) { Return(&ec2.CreateTagsOutput{}, nil) }, }, + { + name: "IPv6 enabled vpc with default subnets should succeed", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + }, + Subnets: []infrav1.SubnetSpec{ + { + ID: "subnet-1", + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a03::/64", + }, + { + ID: "subnet-2", + IsIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a02::/64", + }, + }, + }), + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeSubnets(gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{ + Subnets: []*ec2.Subnet{ + { + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + AvailabilityZone: aws.String("us-east-1a"), + CidrBlock: aws.String("10.0.10.0/24"), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a01::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + MapPublicIpOnLaunch: aws.Bool(false), + AssignIpv6AddressOnCreation: aws.Bool(true), + }, + { + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + AvailabilityZone: aws.String("us-east-1a"), + CidrBlock: aws.String("10.0.20.0/24"), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + MapPublicIpOnLaunch: aws.Bool(false), + AssignIpv6AddressOnCreation: aws.Bool(true), + }, + }, + }, nil) + + m.DescribeRouteTables(gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{ + RouteTables: []*ec2.RouteTable{ + { + VpcId: aws.String(subnetsVPCID), + Associations: []*ec2.RouteTableAssociation{ + { + SubnetId: aws.String("subnet-1"), + RouteTableId: aws.String("rt-12345"), + }, + }, + Routes: []*ec2.Route{ + { + GatewayId: aws.String("igw-12345"), + }, + }, + }, + }, + }, nil) + + m.DescribeNatGatewaysPages( + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.CreateTags(gomock.Eq(&ec2.CreateTagsInput{ + Resources: aws.StringSlice([]string{"subnet-1"}), + Tags: []*ec2.Tag{ + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + }, + })). + Return(&ec2.CreateTagsOutput{}, nil) + + m.CreateTags(gomock.Eq(&ec2.CreateTagsInput{ + Resources: aws.StringSlice([]string{"subnet-2"}), + Tags: []*ec2.Tag{ + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + }, + })). + Return(&ec2.CreateTagsOutput{}, nil) + }, + }, { name: "Unmanaged VPC, 2 existing subnets in vpc, 2 subnet in spec, subnets match, no routes, should succeed", input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ @@ -935,6 +1072,205 @@ func TestReconcileSubnets(t *testing.T) { After(secondSubnet) }, }, + { + name: "Managed IPv6 VPC, no existing subnets exist, one az, expect one private and one public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a01::/56", + IPv6Pool: "amazon", + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeAvailabilityZones(gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + }, + }, + }, nil) + + describeCall := m.DescribeSubnets(gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTables(gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPages( + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + firstSubnet := m.CreateSubnet(gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.0.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailable(gomock.Any()). + After(firstSubnet) + + m.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + secondSubnet := m.CreateSubnet(gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.128.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(firstSubnet) + + m.WaitUntilSubnetAvailable(gomock.Any()). + After(secondSubnet) + }, + }, { name: "Managed VPC, no existing subnets exist, two az's, expect two private and two public from default", input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ diff --git a/pkg/cloud/services/network/vpc.go b/pkg/cloud/services/network/vpc.go index a5947145e9..9749065899 100644 --- a/pkg/cloud/services/network/vpc.go +++ b/pkg/cloud/services/network/vpc.go @@ -52,6 +52,9 @@ func (s *Service) reconcileVPC() error { s.scope.VPC().CidrBlock = vpc.CidrBlock s.scope.VPC().Tags = vpc.Tags + s.scope.VPC().EnableIPv6 = vpc.EnableIPv6 + s.scope.VPC().IPv6CidrBlock = vpc.IPv6CidrBlock + s.scope.VPC().IPv6Pool = vpc.IPv6Pool // If VPC is unmanaged, return early. if vpc.IsUnmanaged(s.scope.Name()) { @@ -90,6 +93,9 @@ func (s *Service) reconcileVPC() error { s.scope.Info("Created VPC", "vpc-id", vpc.ID) s.scope.VPC().CidrBlock = vpc.CidrBlock + s.scope.VPC().IPv6CidrBlock = vpc.IPv6CidrBlock + s.scope.VPC().IPv6Pool = vpc.IPv6Pool + s.scope.VPC().EnableIPv6 = vpc.EnableIPv6 s.scope.VPC().Tags = vpc.Tags s.scope.VPC().ID = vpc.ID @@ -172,17 +178,26 @@ func (s *Service) ensureManagedVPCAttributes(vpc *infrav1.VPCSpec) error { } func (s *Service) createVPC() (*infrav1.VPCSpec, error) { - if s.scope.VPC().CidrBlock == "" { - s.scope.VPC().CidrBlock = defaultVPCCidr - } - input := &ec2.CreateVpcInput{ - CidrBlock: aws.String(s.scope.VPC().CidrBlock), TagSpecifications: []*ec2.TagSpecification{ tags.BuildParamsToTagSpecification(ec2.ResourceTypeVpc, s.getVPCTagParams(services.TemporaryResourceID)), }, } + // setup BYOIP + if s.scope.VPC().IPv6CidrBlock != "" { + input.Ipv6CidrBlock = aws.String(s.scope.VPC().IPv6CidrBlock) + input.Ipv6Pool = aws.String(s.scope.VPC().IPv6Pool) + input.AmazonProvidedIpv6CidrBlock = aws.Bool(false) + } else { + input.AmazonProvidedIpv6CidrBlock = aws.Bool(s.scope.VPC().EnableIPv6) + } + + if s.scope.VPC().CidrBlock == "" { + s.scope.VPC().CidrBlock = defaultVPCCidr + } + input.CidrBlock = &s.scope.VPC().CidrBlock + out, err := s.EC2Client.CreateVpc(input) if err != nil { record.Warnf(s.scope.InfraCluster(), "FailedCreateVPC", "Failed to create new managed VPC: %v", err) @@ -192,11 +207,52 @@ func (s *Service) createVPC() (*infrav1.VPCSpec, error) { record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateVPC", "Created new managed VPC %q", *out.Vpc.VpcId) s.scope.V(2).Info("Created new VPC with cidr", "vpc-id", *out.Vpc.VpcId, "cidr-block", *out.Vpc.CidrBlock) - return &infrav1.VPCSpec{ - ID: *out.Vpc.VpcId, - CidrBlock: *out.Vpc.CidrBlock, - Tags: converters.TagsToMap(out.Vpc.Tags), - }, nil + if !s.scope.VPC().EnableIPv6 { + return &infrav1.VPCSpec{ + ID: *out.Vpc.VpcId, + CidrBlock: *out.Vpc.CidrBlock, + Tags: converters.TagsToMap(out.Vpc.Tags), + }, nil + } + + // BYOIP was defined, no need to look up the VPC. + if s.scope.VPC().EnableIPv6 && s.scope.VPC().IPv6CidrBlock != "" { + return &infrav1.VPCSpec{ + ID: *out.Vpc.VpcId, + CidrBlock: *out.Vpc.CidrBlock, + EnableIPv6: true, + IPv6CidrBlock: s.scope.VPC().IPv6CidrBlock, + IPv6Pool: s.scope.VPC().IPv6Pool, + Tags: converters.TagsToMap(out.Vpc.Tags), + }, nil + } + + // We have to describe the VPC again because the `create` output will **NOT** contain the associated IPv6 address. + vpc, err := s.EC2Client.DescribeVpcs(&ec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{aws.StringValue(out.Vpc.VpcId)}), + }) + if err != nil { + record.Warnf(s.scope.InfraCluster(), "DescribeVpcs", "Failed to describe the new ipv6 vpc: %v", err) + return nil, errors.Wrap(err, "failed to describe new ipv6 vpc") + } + if len(vpc.Vpcs) == 0 { + record.Warnf(s.scope.InfraCluster(), "DescribeVpcs", "Failed to find the new ipv6 vpc, returned list was empty.") + return nil, errors.New("failed to find new ipv6 vpc; returned list was empty") + } + for _, set := range vpc.Vpcs[0].Ipv6CidrBlockAssociationSet { + if *set.Ipv6CidrBlockState.State == ec2.SubnetCidrBlockStateCodeAssociated { + return &infrav1.VPCSpec{ + EnableIPv6: true, + ID: *vpc.Vpcs[0].VpcId, + CidrBlock: *out.Vpc.CidrBlock, + IPv6CidrBlock: aws.StringValue(set.Ipv6CidrBlock), + IPv6Pool: aws.StringValue(set.Ipv6Pool), + Tags: converters.TagsToMap(vpc.Vpcs[0].Tags), + }, nil + } + } + + return nil, fmt.Errorf("no IPv6 associated CIDR block sets found for IPv6 enabled cluster with vpc id %s", *out.Vpc.VpcId) } func (s *Service) deleteVPC() error { @@ -217,6 +273,13 @@ func (s *Service) deleteVPC() error { s.scope.V(4).Info("Skipping VPC deletion, VPC not found") return nil } + + // Ignore if VPC ID is not present, + if code, ok := awserrors.Code(err); ok && code == awserrors.VPCMissingParameter { + s.scope.V(4).Info("Skipping VPC deletion, VPC ID not present") + return nil + } + record.Warnf(s.scope.InfraCluster(), "FailedDeleteVPC", "Failed to delete managed VPC %q: %v", vpc.ID, err) return errors.Wrapf(err, "failed to delete vpc %q", vpc.ID) } @@ -260,11 +323,20 @@ func (s *Service) describeVPCByID() (*infrav1.VPCSpec, error) { return nil, awserrors.NewNotFound("could not find available or pending vpc") } - return &infrav1.VPCSpec{ + vpc := &infrav1.VPCSpec{ ID: *out.Vpcs[0].VpcId, CidrBlock: *out.Vpcs[0].CidrBlock, Tags: converters.TagsToMap(out.Vpcs[0].Tags), - }, nil + } + for _, set := range out.Vpcs[0].Ipv6CidrBlockAssociationSet { + if *set.Ipv6CidrBlockState.State == ec2.SubnetCidrBlockStateCodeAssociated { + vpc.IPv6CidrBlock = aws.StringValue(set.Ipv6CidrBlock) + vpc.IPv6Pool = aws.StringValue(set.Ipv6Pool) + vpc.EnableIPv6 = true + break + } + } + return vpc, nil } func (s *Service) getVPCTagParams(id string) infrav1.BuildParams { diff --git a/pkg/cloud/services/network/vpc_test.go b/pkg/cloud/services/network/vpc_test.go index 243a332f54..6eee844b83 100644 --- a/pkg/cloud/services/network/vpc_test.go +++ b/pkg/cloud/services/network/vpc_test.go @@ -163,6 +163,238 @@ func TestReconcileVPC(t *testing.T) { m.ModifyVpcAttribute(gomock.AssignableToTypeOf(&ec2.ModifyVpcAttributeInput{})).Return(&ec2.ModifyVpcAttributeOutput{}, nil).Times(2) }, }, + { + name: "Should create a new IPv6 VPC if managed IPv6 vpc does not exist", + input: &infrav1.VPCSpec{ + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + EnableIPv6: true, + }, + wantErr: false, + want: &infrav1.VPCSpec{ + ID: "vpc-new", + CidrBlock: "10.1.0.0/16", + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a03::/56", + IPv6Pool: "amazon", + Tags: map[string]string{ + "sigs.k8s.io/cluster-api-provider-aws/role": "common", + "Name": "test-cluster-vpc", + "sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster": "owned", + }, + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + }, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeVpcs(gomock.AssignableToTypeOf(&ec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{"vpc-new"}), + })).Return(&ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{ + { + CidrBlock: aws.String("10.1.0.0/16"), + Ipv6CidrBlockAssociationSet: []*ec2.VpcIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/56"), + Ipv6CidrBlockState: &ec2.VpcCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + Ipv6Pool: aws.String("amazon"), + }, + }, + State: aws.String("available"), + Tags: tags, + VpcId: aws.String("vpc-new"), + }, + }, + }, nil) + m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{ + AmazonProvidedIpv6CidrBlock: aws.Bool(true), + })).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.1.0.0/16"), + Tags: tags, + }, + }, nil) + + m.DescribeVpcAttribute(gomock.AssignableToTypeOf(&ec2.DescribeVpcAttributeInput{})). + DoAndReturn(describeVpcAttributeFalse).MinTimes(1) + + m.ModifyVpcAttribute(gomock.AssignableToTypeOf(&ec2.ModifyVpcAttributeInput{})).Return(&ec2.ModifyVpcAttributeOutput{}, nil).Times(2) + }, + }, + { + name: "Should create a new IPv6 VPC with BYOIP set up if managed IPv6 vpc does not exist", + input: &infrav1.VPCSpec{ + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a03::/56", + IPv6Pool: "my-pool", + }, + wantErr: false, + want: &infrav1.VPCSpec{ + ID: "vpc-new", + CidrBlock: "10.1.0.0/16", + EnableIPv6: true, + IPv6CidrBlock: "2001:db8:1234:1a03::/56", + IPv6Pool: "my-pool", + Tags: map[string]string{ + "sigs.k8s.io/cluster-api-provider-aws/role": "common", + "Name": "test-cluster-vpc", + "sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster": "owned", + }, + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + }, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{ + AmazonProvidedIpv6CidrBlock: aws.Bool(false), + Ipv6Pool: aws.String("my-pool"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/56"), + })).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.1.0.0/16"), + Tags: tags, + }, + }, nil) + + m.DescribeVpcAttribute(gomock.AssignableToTypeOf(&ec2.DescribeVpcAttributeInput{})). + DoAndReturn(describeVpcAttributeFalse).MinTimes(1) + + m.ModifyVpcAttribute(gomock.AssignableToTypeOf(&ec2.ModifyVpcAttributeInput{})).Return(&ec2.ModifyVpcAttributeOutput{}, nil).Times(2) + }, + }, + { + name: "Describing the VPC fails with IPv6 VPC should return an error", + input: &infrav1.VPCSpec{ + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + EnableIPv6: true, + }, + wantErr: true, + want: nil, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeVpcs(gomock.AssignableToTypeOf(&ec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{"vpc-new"}), + })).Return(nil, errors.New("nope")) + m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{})).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.1.0.0/16"), + Tags: tags, + }, + }, nil) + }, + }, + { + name: "Describing an IPv6 VPC returns no results should return an error", + input: &infrav1.VPCSpec{ + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + EnableIPv6: true, + }, + wantErr: true, + want: nil, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeVpcs(gomock.AssignableToTypeOf(&ec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{"vpc-new"}), + })).Return(&ec2.DescribeVpcsOutput{}, nil) + m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{})).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.1.0.0/16"), + Tags: tags, + }, + }, nil) + }, + }, + { + name: "Describing an IPv6 VPC without ipv6 cidr associations should return an error", + input: &infrav1.VPCSpec{ + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + EnableIPv6: true, + }, + wantErr: true, + want: nil, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeVpcs(gomock.AssignableToTypeOf(&ec2.DescribeVpcsInput{ + VpcIds: aws.StringSlice([]string{"vpc-new"}), + })).Return(&ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{ + { + CidrBlock: aws.String("10.1.0.0/16"), + State: aws.String("available"), + Tags: tags, + VpcId: aws.String("vpc-new"), + }, + }, + }, nil) + m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{})).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.1.0.0/16"), + Tags: tags, + }, + }, nil) + }, + }, + { + name: "should set up IPv6 associations if found VPC is IPv6 enabled", + input: &infrav1.VPCSpec{ID: "unmanaged-vpc-exists", AvailabilityZoneUsageLimit: &usageLimit, AvailabilityZoneSelection: &selection}, + want: &infrav1.VPCSpec{ + ID: "unmanaged-vpc-exists", + CidrBlock: "10.0.0.0/8", + Tags: map[string]string{ + "sigs.k8s.io/cluster-api-provider-aws/role": "common", + "Name": "test-cluster-vpc", + }, + IPv6Pool: "my-pool", + IPv6CidrBlock: "2001:db8:1234:1a03::/56", + EnableIPv6: true, + AvailabilityZoneUsageLimit: &usageLimit, + AvailabilityZoneSelection: &selection, + }, + expect: func(m *mock_ec2iface.MockEC2APIMockRecorder) { + m.DescribeVpcs(gomock.AssignableToTypeOf(&ec2.DescribeVpcsInput{})).Return(&ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{ + { + State: aws.String("available"), + VpcId: aws.String("unmanaged-vpc-exists"), + CidrBlock: aws.String("10.0.0.0/8"), + Ipv6CidrBlockAssociationSet: []*ec2.VpcIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/56"), + Ipv6CidrBlockState: &ec2.VpcCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + Ipv6Pool: aws.String("my-pool"), + }, + }, + Tags: []*ec2.Tag{ + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-vpc"), + }, + }, + }, + }, + }, nil) + }, + }, { name: "managed vpc id exists, but vpc resource is missing", input: &infrav1.VPCSpec{ID: "vpc-exists", AvailabilityZoneUsageLimit: &usageLimit, AvailabilityZoneSelection: &selection}, diff --git a/pkg/cloud/services/s3/s3.go b/pkg/cloud/services/s3/s3.go index e23609f66c..48b85d0b2d 100644 --- a/pkg/cloud/services/s3/s3.go +++ b/pkg/cloud/services/s3/s3.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" "net/url" - "path" + "sigs.k8s.io/cluster-api-provider-aws/util/system" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -44,6 +44,24 @@ type Service struct { STSClient stsiface.STSAPI } +var DisabledError = errors.New("s3 management disabled") + +func IsDisabledError(err error) bool { + return err == DisabledError +} + +var EmptyBucketError = errors.New("empty bucket name") + +func IsEmptyBucketError(err error) bool { + return err == EmptyBucketError +} + +var EmptyKeyError = errors.New("empty key") + +func IsEmptyKeyError(err error) bool { + return err == EmptyKeyError +} + // NewService returns a new service given the api clients. func NewService(s3Scope scope.S3Scope) *Service { s3Client := scope.NewS3Client(s3Scope, s3Scope, s3Scope, s3Scope.InfraCluster()) @@ -80,6 +98,9 @@ func (s *Service) DeleteBucket() error { } bucketName := s.bucketName() + if bucketName == "" { + return EmptyBucketError + } log := s.scope.WithValues("name", bucketName) @@ -109,58 +130,78 @@ func (s *Service) DeleteBucket() error { return nil } -func (s *Service) Create(m *scope.MachineScope, data []byte) (string, error) { +// Create will add a file to the s3 bucket which is private and server side encrypted. +func (s *Service) Create(key string, data []byte) (string, error) { if !s.bucketManagementEnabled() { - return "", errors.New("requested object creation but bucket management is not enabled") + return "", DisabledError } - if m == nil { - return "", errors.New("machine scope can't be nil") + // server side encryption, acl defaults to private + return s.create(&s3.PutObjectInput{ + Body: aws.ReadSeekCloser(bytes.NewReader(data)), + Bucket: aws.String(s.scope.Bucket().Name), + Key: aws.String(key), + ServerSideEncryption: aws.String("aws:kms"), + }) +} + +// CreatePublic will add file to the s3 bucket which is public and open to the world to access. +func (s *Service) CreatePublic(key string, data []byte) (string, error) { + // acl public-read + if !s.bucketManagementEnabled() { + return "", DisabledError } - if len(data) == 0 { - return "", errors.New("got empty data") + return s.create(&s3.PutObjectInput{ + Body: aws.ReadSeekCloser(bytes.NewReader(data)), + Bucket: aws.String(s.scope.Bucket().Name), + Key: aws.String(key), + ACL: aws.String("public-read"), + }) +} + +func (s *Service) create(putInput *s3.PutObjectInput) (string, error) { + if aws.StringValue(putInput.Bucket) == "" { + return "", EmptyBucketError } - bucket := s.bucketName() - key := s.bootstrapDataKey(m) + if aws.StringValue(putInput.Key) == "" { + return "", EmptyKeyError + } - s.scope.Info("Creating object", "bucket_name", bucket, "key", key) + s.scope.Info("Creating public object", "bucket_name", aws.StringValue(putInput.Bucket), "key", aws.StringValue(putInput.Key)) - if _, err := s.S3Client.PutObject(&s3.PutObjectInput{ - Body: aws.ReadSeekCloser(bytes.NewReader(data)), - Bucket: aws.String(bucket), - Key: aws.String(key), - ServerSideEncryption: aws.String("aws:kms"), - }); err != nil { + if _, err := s.S3Client.PutObject(putInput); err != nil { return "", errors.Wrap(err, "putting object") } objectURL := &url.URL{ Scheme: "s3", - Host: bucket, - Path: key, + Host: aws.StringValue(putInput.Bucket), + Path: aws.StringValue(putInput.Key), } return objectURL.String(), nil } -func (s *Service) Delete(m *scope.MachineScope) error { +func (s *Service) Delete(key string) error { if !s.bucketManagementEnabled() { - return errors.New("requested object creation but bucket management is not enabled") + return DisabledError } - if m == nil { - return errors.New("machine scope can't be nil") + if key == "" { + return EmptyKeyError } - bucket := s.bucketName() - key := s.bootstrapDataKey(m) + bucketName := s.bucketName() + if bucketName == "" { + return EmptyBucketError + } - s.scope.Info("Deleting object", "bucket_name", bucket, "key", key) + s.scope.Info("Deleting object", "bucket_name", bucketName, "key", key) _, err := s.S3Client.DeleteObject(&s3.DeleteObjectInput{ - Bucket: aws.String(bucket), + Bucket: aws.String(bucketName), Key: aws.String(key), }) if err == nil { @@ -236,16 +277,17 @@ func (s *Service) bucketPolicy(bucketName string) (string, error) { } bucket := s.scope.Bucket() + partition := system.GetPartitionFromRegion(s.scope.Region()) statements := []iam.StatementEntry{ { Sid: "control-plane", Effect: iam.EffectAllow, Principal: map[iam.PrincipalType]iam.PrincipalID{ - iam.PrincipalAWS: []string{fmt.Sprintf("arn:aws:iam::%s:role/%s", *accountID.Account, bucket.ControlPlaneIAMInstanceProfile)}, + iam.PrincipalAWS: []string{fmt.Sprintf("arn:%s:iam::%s:role/%s", s.scope, *accountID.Account, bucket.ControlPlaneIAMInstanceProfile)}, }, Action: []string{"s3:GetObject"}, - Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/control-plane/*", bucketName)}, + Resource: []string{fmt.Sprintf("arn:%s:s3:::%s/control-plane/*", partition, bucketName)}, }, } @@ -254,10 +296,10 @@ func (s *Service) bucketPolicy(bucketName string) (string, error) { Sid: iamInstanceProfile, Effect: iam.EffectAllow, Principal: map[iam.PrincipalType]iam.PrincipalID{ - iam.PrincipalAWS: []string{fmt.Sprintf("arn:aws:iam::%s:role/%s", *accountID.Account, iamInstanceProfile)}, + iam.PrincipalAWS: []string{fmt.Sprintf("arn:%s:iam::%s:role/%s", partition, *accountID.Account, iamInstanceProfile)}, }, Action: []string{"s3:GetObject"}, - Resource: []string{fmt.Sprintf("arn:aws:s3:::%s/node/*", bucketName)}, + Resource: []string{fmt.Sprintf("arn:%s:s3:::%s/node/*", partition, bucketName)}, }) } @@ -281,8 +323,3 @@ func (s *Service) bucketManagementEnabled() bool { func (s *Service) bucketName() string { return s.scope.Bucket().Name } - -func (s *Service) bootstrapDataKey(m *scope.MachineScope) string { - // Use machine name as object key. - return path.Join(m.Role(), m.Name()) -} diff --git a/pkg/cloud/services/securitygroup/securitygroups.go b/pkg/cloud/services/securitygroup/securitygroups.go index a3feb3e7dd..207ce4a7d1 100644 --- a/pkg/cloud/services/securitygroup/securitygroups.go +++ b/pkg/cloud/services/securitygroup/securitygroups.go @@ -475,7 +475,10 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( }, } } - + cidrBlocks := []string{services.AnyIPv4CidrBlock} + if s.scope.VPC().EnableIPv6 { + cidrBlocks = append(cidrBlocks, services.AnyIPv6CidrBlock) + } switch role { case infrav1.SecurityGroupBastion: return infrav1.IngressRules{ @@ -527,7 +530,7 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( Protocol: infrav1.SecurityGroupProtocolTCP, FromPort: 30000, ToPort: 32767, - CidrBlocks: []string{services.AnyIPv4CidrBlock}, + CidrBlocks: cidrBlocks, }, { Description: "Kubelet API", @@ -559,7 +562,7 @@ func (s *Service) getSecurityGroupIngressRules(role infrav1.SecurityGroupRole) ( Protocol: infrav1.SecurityGroupProtocolTCP, FromPort: int64(s.scope.APIServerPort()), ToPort: int64(s.scope.APIServerPort()), - CidrBlocks: []string{services.AnyIPv4CidrBlock}, + CidrBlocks: cidrBlocks, }, }, nil case infrav1.SecurityGroupLB: diff --git a/pkg/eks/identityprovider/plan.go b/pkg/eks/identityprovider/plan.go index 75f6b434d3..240fcf7e36 100644 --- a/pkg/eks/identityprovider/plan.go +++ b/pkg/eks/identityprovider/plan.go @@ -19,7 +19,6 @@ package identityprovider import ( "context" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/go-logr/logr" @@ -58,7 +57,7 @@ func (p *plan) Create(ctx context.Context) ([]planner.Procedure, error) { if p.desiredIdentityProvider == nil { // disassociation will also also trigger deletion hence // we do nothing in case of ConfigStatusDeleting as it will happen eventually - if aws.StringValue(p.currentIdentityProvider.Status) == eks.ConfigStatusActive { + if p.currentIdentityProvider.Status == eks.ConfigStatusActive { procedures = append(procedures, &DisassociateIdentityProviderConfig{plan: p}) } @@ -80,7 +79,7 @@ func (p *plan) Create(ctx context.Context) ([]planner.Procedure, error) { if len(p.desiredIdentityProvider.Tags) == 0 && len(p.currentIdentityProvider.Tags) != 0 { procedures = append(procedures, &RemoveIdentityProviderTagsProcedure{plan: p}) } - switch aws.StringValue(p.currentIdentityProvider.Status) { + switch p.currentIdentityProvider.Status { case eks.ConfigStatusActive: // config active no work to be done return procedures, nil diff --git a/pkg/eks/identityprovider/plan_test.go b/pkg/eks/identityprovider/plan_test.go index 5c4309999c..5156771c22 100644 --- a/pkg/eks/identityprovider/plan_test.go +++ b/pkg/eks/identityprovider/plan_test.go @@ -227,8 +227,8 @@ func createDesiredIdentityProvider(name string, tags infrav1.Tags) *OidcIdentity func createCurrentIdentityProvider(name string, arn, status string, tags infrav1.Tags) *OidcIdentityProviderConfig { config := createDesiredIdentityProvider(name, tags) - config.IdentityProviderConfigArn = aws.String(arn) - config.Status = aws.String(status) + config.IdentityProviderConfigArn = arn + config.Status = status return config } @@ -243,6 +243,11 @@ func createDesiredIdentityProviderRequest(name *string) *eks.OidcIdentityProvide ClientId: aws.String("clientId"), IdentityProviderConfigName: name, IssuerUrl: aws.String("http://IssuerURL.com"), + RequiredClaims: make(map[string]*string), + GroupsClaim: aws.String(""), + GroupsPrefix: aws.String(""), + UsernameClaim: aws.String(""), + UsernamePrefix: aws.String(""), } } diff --git a/pkg/eks/identityprovider/procedures.go b/pkg/eks/identityprovider/procedures.go index dd617046f3..6338f6620b 100644 --- a/pkg/eks/identityprovider/procedures.go +++ b/pkg/eks/identityprovider/procedures.go @@ -106,13 +106,13 @@ func (a *AssociateIdentityProviderProcedure) Do(ctx context.Context) error { ClusterName: aws.String(a.plan.clusterName), Oidc: &eks.OidcIdentityProviderConfigRequest{ ClientId: aws.String(oidc.ClientID), - GroupsClaim: oidc.GroupsClaim, - GroupsPrefix: oidc.GroupsPrefix, + GroupsClaim: aws.String(oidc.GroupsClaim), + GroupsPrefix: aws.String(oidc.GroupsPrefix), IdentityProviderConfigName: aws.String(oidc.IdentityProviderConfigName), IssuerUrl: aws.String(oidc.IssuerURL), - RequiredClaims: oidc.RequiredClaims, - UsernameClaim: oidc.UsernameClaim, - UsernamePrefix: oidc.UsernamePrefix, + RequiredClaims: aws.StringMap(oidc.RequiredClaims), + UsernameClaim: aws.String(oidc.UsernameClaim), + UsernamePrefix: aws.String(oidc.UsernamePrefix), }, } @@ -139,7 +139,7 @@ func (u *UpdatedIdentityProviderTagsProcedure) Name() string { func (u *UpdatedIdentityProviderTagsProcedure) Do(ctx context.Context) error { arn := u.plan.currentIdentityProvider.IdentityProviderConfigArn _, err := u.plan.eksClient.TagResource(&eks.TagResourceInput{ - ResourceArn: arn, + ResourceArn: &arn, Tags: aws.StringMap(u.plan.desiredIdentityProvider.Tags), }) @@ -164,8 +164,10 @@ func (r *RemoveIdentityProviderTagsProcedure) Do(ctx context.Context) error { for key := range r.plan.currentIdentityProvider.Tags { keys = append(keys, aws.String(key)) } + + arn := r.plan.currentIdentityProvider.IdentityProviderConfigArn _, err := r.plan.eksClient.UntagResource(&eks.UntagResourceInput{ - ResourceArn: r.plan.currentIdentityProvider.IdentityProviderConfigArn, + ResourceArn: &arn, TagKeys: keys, }) diff --git a/pkg/eks/identityprovider/types.go b/pkg/eks/identityprovider/types.go index 5d4c528038..a5afec9a87 100644 --- a/pkg/eks/identityprovider/types.go +++ b/pkg/eks/identityprovider/types.go @@ -17,25 +17,26 @@ limitations under the License. package identityprovider import ( - "github.com/google/go-cmp/cmp" + "reflect" infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1" ) -// OidcIdentityProviderConfig represents the configuration for an OpenID Connect (OIDC) -// identity provider. +// OidcIdentityProviderConfig represents a normalized version of the configuration for an OpenID Connect (OIDC) +// identity provider configuration. To reconcile the config we are going to get the version from EKS and +// AWSManagedControlPlane and will need to have one consistent version of string values from each API. type OidcIdentityProviderConfig struct { ClientID string - GroupsClaim *string - GroupsPrefix *string - IdentityProviderConfigArn *string + GroupsClaim string + GroupsPrefix string + IdentityProviderConfigArn string IdentityProviderConfigName string IssuerURL string - RequiredClaims map[string]*string - Status *string + RequiredClaims map[string]string + Status string Tags infrav1.Tags - UsernameClaim *string - UsernamePrefix *string + UsernameClaim string + UsernamePrefix string } func (o *OidcIdentityProviderConfig) IsEqual(other *OidcIdentityProviderConfig) bool { @@ -43,35 +44,35 @@ func (o *OidcIdentityProviderConfig) IsEqual(other *OidcIdentityProviderConfig) return true } - if !cmp.Equal(o.ClientID, other.ClientID) { + if o.ClientID != other.ClientID { return false } - if !cmp.Equal(o.GroupsClaim, other.GroupsClaim) { + if o.GroupsClaim != other.GroupsClaim { return false } - if !cmp.Equal(o.GroupsPrefix, other.GroupsPrefix) { + if o.GroupsPrefix != other.GroupsPrefix { return false } - if !cmp.Equal(o.IdentityProviderConfigName, other.IdentityProviderConfigName) { + if o.IdentityProviderConfigName != other.IdentityProviderConfigName { return false } - if !cmp.Equal(o.IssuerURL, other.IssuerURL) { + if o.IssuerURL != other.IssuerURL { return false } - if !cmp.Equal(o.RequiredClaims, other.RequiredClaims) { + if !reflect.DeepEqual(o.RequiredClaims, other.RequiredClaims) { return false } - if !cmp.Equal(o.UsernameClaim, other.UsernameClaim) { + if o.UsernameClaim != other.UsernameClaim { return false } - if !cmp.Equal(o.UsernamePrefix, other.UsernamePrefix) { + if o.UsernamePrefix != other.UsernamePrefix { return false } diff --git a/pkg/eks/identityprovider/types_test.go b/pkg/eks/identityprovider/types_test.go index ed2f593e28..c172758f8a 100644 --- a/pkg/eks/identityprovider/types_test.go +++ b/pkg/eks/identityprovider/types_test.go @@ -19,7 +19,6 @@ package identityprovider import ( "testing" - "github.com/aws/aws-sdk-go/aws" "github.com/onsi/gomega" ) @@ -39,7 +38,7 @@ func TestIdentityProviderEqual(t *testing.T) { ClientID: "a", IdentityProviderConfigName: "b", IssuerURL: "c", - Status: aws.String("e"), + Status: "e", }, }, } diff --git a/pkg/internal/cidr/cidr.go b/pkg/internal/cidr/cidr.go index 93396b526b..dd56ee5e75 100644 --- a/pkg/internal/cidr/cidr.go +++ b/pkg/internal/cidr/cidr.go @@ -26,8 +26,8 @@ import ( ) // SplitIntoSubnetsIPv4 splits a IPv4 CIDR into a specified number of subnets. -// If the number of required subnets isn't a power of 2 then then CIDR will be split -// into the the next highest power of 2 and you will end up with unused ranges. +// If the number of required subnets isn't a power of 2 then CIDR will be split +// into the next highest power of 2, and you will end up with unused ranges. // NOTE: this code is adapted from kops https://github.com/kubernetes/kops/blob/c323819e6480d71bad8d21184516e3162eaeca8f/pkg/util/subnet/subnet.go#L46 func SplitIntoSubnetsIPv4(cidrBlock string, numSubnets int) ([]*net.IPNet, error) { _, parent, err := net.ParseCIDR(cidrBlock) @@ -65,6 +65,44 @@ func SplitIntoSubnetsIPv4(cidrBlock string, numSubnets int) ([]*net.IPNet, error return subnets, nil } +const subnetIDLocation = 7 + +// SplitIntoSubnetsIPv6 splits a IPv6 address into a specified number of subnets. +// AWS IPv6 based subnets **must always have a /64 prefix**. AWS provides an IPv6 +// CIDR with /56 prefix. That's the initial CIDR. We must convert that to /64 and +// slice the subnets by increasing the subnet ID by 1. +// so given: 2600:1f14:e08:7400::/56 +// sub1: 2600:1f14:e08:7400::/64 +// sub2: 2600:1f14:e08:7401::/64 +// sub3: 2600:1f14:e08:7402::/64 +// sub4: 2600:1f14:e08:7403::/64 +// This function can also be called with /64 prefix to further slice existing subnet +// addresses. +// When splitting further, we always have to take the LAST one to avoid collisions +// since the prefix stays the same, but the subnet ID increases. +// To see this restriction read https://docs.aws.amazon.com/vpc/latest/userguide/how-it-works.html#ipv4-ipv6-comparison +func SplitIntoSubnetsIPv6(cidrBlock string, numSubnets int) ([]*net.IPNet, error) { + _, ipv6CidrBlock, err := net.ParseCIDR(cidrBlock) + if err != nil { + return nil, fmt.Errorf("failed to parse cidr block %s with error: %w", cidrBlock, err) + } + // update the prefix to 64. + ipv6CidrBlock.Mask = net.CIDRMask(64, 128) + var ( + subnets []*net.IPNet + ) + for i := 0; i < numSubnets; i++ { + ipv6CidrBlock.IP[subnetIDLocation]++ + newIP := net.ParseIP(ipv6CidrBlock.IP.String()) + v := &net.IPNet{ + IP: newIP, + Mask: net.CIDRMask(64, 128), + } + subnets = append(subnets, v) + } + return subnets, nil +} + // GetIPv4Cidrs gets the IPv4 CIDRs from a string slice. func GetIPv4Cidrs(cidrs []string) ([]string, error) { found := []string{} diff --git a/pkg/internal/cidr/cidr_test.go b/pkg/internal/cidr/cidr_test.go index df1bed88d2..5b1ef3ee28 100644 --- a/pkg/internal/cidr/cidr_test.go +++ b/pkg/internal/cidr/cidr_test.go @@ -24,6 +24,10 @@ import ( "sigs.k8s.io/cluster-api-provider-aws/pkg/internal/cidr" ) +var ( + block = "2001:db8:1234:1a00::/56" +) + func TestParseIPv4CIDR(t *testing.T) { RegisterTestingT(t) @@ -51,3 +55,145 @@ func TestParseIPv6CIDR(t *testing.T) { Expect(err).NotTo(HaveOccurred()) Expect(output).To(HaveLen(2)) } + +func TestSplitIntoSubnetsIPv6(t *testing.T) { + RegisterTestingT(t) + ip1, _, _ := net.ParseCIDR("2001:db8:1234:1a01::/64") + ip2, _, _ := net.ParseCIDR("2001:db8:1234:1a02::/64") + ip3, _, _ := net.ParseCIDR("2001:db8:1234:1a03::/64") + ip4, _, _ := net.ParseCIDR("2001:db8:1234:1a04::/64") + output, err := SplitIntoSubnetsIPv6(block, 4) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ConsistOf( + &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip3, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip4, + Mask: net.CIDRMask(64, 128), + }, + )) +} + +func TestSplitIntoSubnetsIPv6WithFurtherSplitting(t *testing.T) { + RegisterTestingT(t) + ip1, _, _ := net.ParseCIDR("2001:db8:1234:1a01::/64") + ip2, _, _ := net.ParseCIDR("2001:db8:1234:1a02::/64") + ip3, _, _ := net.ParseCIDR("2001:db8:1234:1a03::/64") + ip4, _, _ := net.ParseCIDR("2001:db8:1234:1a04::/64") + output, err := SplitIntoSubnetsIPv6(block, 4) + Expect(err).NotTo(HaveOccurred()) + Expect(output).To(ConsistOf( + &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip3, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip4, + Mask: net.CIDRMask(64, 128), + }, + )) + output, err = SplitIntoSubnetsIPv6(output[len(output)-1].String(), 3) + Expect(err).NotTo(HaveOccurred()) + ip1, _, _ = net.ParseCIDR("2001:db8:1234:1a05::/64") + ip2, _, _ = net.ParseCIDR("2001:db8:1234:1a06::/64") + ip3, _, _ = net.ParseCIDR("2001:db8:1234:1a07::/64") + Expect(output).To(ContainElements( + &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip3, + Mask: net.CIDRMask(64, 128), + }, + )) +} + +func TestSplitIntoSubnetsIPv6HigherSubnetSplitting(t *testing.T) { + RegisterTestingT(t) + output, err := SplitIntoSubnetsIPv6("2001:db8:cad:ffff::/56", 6) + Expect(err).NotTo(HaveOccurred()) + ip1, _, _ := net.ParseCIDR("2001:db8:cad:ff01::/64") + ip2, _, _ := net.ParseCIDR("2001:db8:cad:ff02::/64") + ip3, _, _ := net.ParseCIDR("2001:db8:cad:ff03::/64") + ip4, _, _ := net.ParseCIDR("2001:db8:cad:ff04::/64") + Expect(output).To(ContainElements( + &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip3, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip4, + Mask: net.CIDRMask(64, 128), + }, + )) +} + +func TestSplitIntoSubnetsIPv6NoCompression(t *testing.T) { + RegisterTestingT(t) + output, err := SplitIntoSubnetsIPv6("2001:0db8:85a3:0010:1111:8a2e:0370:7334/56", 5) + Expect(err).NotTo(HaveOccurred()) + ip1, _, _ := net.ParseCIDR("2001:db8:85a3:1::/64") + ip2, _, _ := net.ParseCIDR("2001:db8:85a3:2::/64") + ip3, _, _ := net.ParseCIDR("2001:db8:85a3:3::/64") + ip4, _, _ := net.ParseCIDR("2001:db8:85a3:4::/64") + ip5, _, _ := net.ParseCIDR("2001:db8:85a3:5::/64") + Expect(output).To(ContainElements( + &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip3, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip4, + Mask: net.CIDRMask(64, 128), + }, + &net.IPNet{ + IP: ip5, + Mask: net.CIDRMask(64, 128), + }, + )) +} + +func TestSplitIntoSubnetsIPv6InvalidCIDR(t *testing.T) { + RegisterTestingT(t) + _, err := SplitIntoSubnetsIPv6("2001:db8:cad::", 60) + Expect(err).To(MatchError(ContainSubstring("failed to parse cidr block 2001:db8:cad:: with error: invalid CIDR address: 2001:db8:cad::"))) +} diff --git a/spectro/base/kustomization.yaml b/spectro/base/kustomization.yaml new file mode 100644 index 0000000000..75930713f0 --- /dev/null +++ b/spectro/base/kustomization.yaml @@ -0,0 +1,44 @@ +namePrefix: capa- +namespace: capa-system + +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-aws" + +resources: + - ../../config/default/credentials.yaml + +bases: + - ../../config/rbac + - ../../config/manager + +patchesStrategicMerge: + - ../../config/default/manager_credentials_patch.yaml + - ../../config/default/manager_service_account_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_image_patch.yaml + +configurations: + - ../../config/default/kustomizeconfig.yaml + +patchesJson6902: + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_healthcheck.yaml + - target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_credentials.yaml \ No newline at end of file diff --git a/spectro/base/patch_credentials.yaml b/spectro/base/patch_credentials.yaml new file mode 100644 index 0000000000..d9a3b55790 --- /dev/null +++ b/spectro/base/patch_credentials.yaml @@ -0,0 +1,3 @@ +- op: replace + path: "/spec/template/spec/volumes/0/secret/secretName" + value: "capa-manager-bootstrap-credentials" \ No newline at end of file diff --git a/spectro/base/patch_healthcheck.yaml b/spectro/base/patch_healthcheck.yaml new file mode 100644 index 0000000000..30acc93e1e --- /dev/null +++ b/spectro/base/patch_healthcheck.yaml @@ -0,0 +1,6 @@ +- op: remove + path: "/spec/template/spec/containers/0/ports" +- op: remove + path: "/spec/template/spec/containers/0/livenessProbe" +- op: remove + path: "/spec/template/spec/containers/0/readinessProbe" diff --git a/spectro/base/patch_service_account.yaml b/spectro/base/patch_service_account.yaml new file mode 100644 index 0000000000..d9cd4321fc --- /dev/null +++ b/spectro/base/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml new file mode 100644 index 0000000000..7b0261dc69 --- /dev/null +++ b/spectro/generated/core-base.yaml @@ -0,0 +1,459 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + ${AWS_CONTROLLER_IAM_ROLE/#arn/eks.amazonaws.com/role-arn: arn} + labels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: controller-manager + name: capa-controller-manager + namespace: capa-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-leader-elect-role + namespace: capa-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-manager-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - bootstrap.cluster.x-k8s.io + resources: + - eksconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - bootstrap.cluster.x-k8s.io + resources: + - eksconfigs/status + verbs: + - get + - patch + - update +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - clusters/status + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - machinepools + - machines + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - machinepools + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - machinepools + - machinepools/status + verbs: + - get + - list + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - machines + - machines/status + verbs: + - get + - list + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes + - awsmanagedcontrolplanes/status + verbs: + - get + - list + - watch +- apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclustercontrolleridentities + verbs: + - create + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclustercontrolleridentities + - awsclusterroleidentities + - awsclusterstaticidentities + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclusterroleidentities + - awsclusterstaticidentities + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclusters/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsfargateprofiles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsfargateprofiles/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachinepools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachinepools + - awsmachinepools/status + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachinepools/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachines + - awsmachines/status + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmachines/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmanagedmachinepools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmanagedmachinepools + - awsmanagedmachinepools/status + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsmanagedmachinepools/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-leader-elect-rolebinding + namespace: capa-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: capa-leader-elect-role +subjects: +- kind: ServiceAccount + name: capa-controller-manager + namespace: capa-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: capa-manager-role +subjects: +- kind: ServiceAccount + name: capa-controller-manager + namespace: capa-system +--- +apiVersion: v1 +data: + credentials: ${AWS_B64ENCODED_CREDENTIALS} +kind: Secret +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-manager-bootstrap-credentials + namespace: capa-system +type: Opaque +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + name: capa-controller-manager + namespace: capa-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: ${K8S_CP_LABEL:=node-role.kubernetes.io/control-plane} + operator: Exists + weight: 10 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + weight: 10 + containers: + - args: + - --leader-elect + - --feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true},BootstrapFormatIgnition=${EXP_BOOTSTRAP_FORMAT_IGNITION:=false},ExternalResourceGC=${EXP_EXTERNAL_RESOURCE_GC:=false} + - --v=${CAPA_LOGLEVEL:=0} + - --metrics-bind-addr=127.0.0.1:8080 + env: + - name: AWS_SHARED_CREDENTIALS_FILE + value: /home/.aws/credentials + image: gcr.io/k8s-staging-cluster-api-aws/cluster-api-aws-controller:latest + imagePullPolicy: Always + name: manager + volumeMounts: + - mountPath: /home/.aws + name: credentials + securityContext: + fsGroup: 1000 + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: credentials + secret: + secretName: capa-manager-bootstrap-credentials diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml new file mode 100644 index 0000000000..efe52bc50a --- /dev/null +++ b/spectro/generated/core-global.yaml @@ -0,0 +1,14346 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + clusterctl.cluster.x-k8s.io/move-hierarchy: "" + name: awsclustercontrolleridentities.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSClusterControllerIdentity + listKind: AWSClusterControllerIdentityList + plural: awsclustercontrolleridentities + shortNames: + - awsci + singular: awsclustercontrolleridentity + scope: Cluster + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSClusterControllerIdentity is the Schema for the awsclustercontrolleridentities + API It is used to grant access to use Cluster API Provider AWS Controller + credentials. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterControllerIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + AllowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSClusterControllerIdentity is the Schema for the awsclustercontrolleridentities + API It is used to grant access to use Cluster API Provider AWS Controller + credentials. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterControllerIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AWSClusterControllerIdentity is the Schema for the awsclustercontrolleridentities + API It is used to grant access to use Cluster API Provider AWS Controller + credentials. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterControllerIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + clusterctl.cluster.x-k8s.io/move-hierarchy: "" + name: awsclusterroleidentities.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSClusterRoleIdentity + listKind: AWSClusterRoleIdentityList + plural: awsclusterroleidentities + shortNames: + - awsri + singular: awsclusterroleidentity + scope: Cluster + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSClusterRoleIdentity is the Schema for the awsclusterroleidentities + API It is used to assume a role using the provided sourceRef. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterRoleIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + AllowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + durationSeconds: + description: The duration, in seconds, of the role session before + it is renewed. + format: int32 + maximum: 43200 + minimum: 900 + type: integer + externalID: + description: A unique identifier that might be required when you assume + a role in another account. If the administrator of the account to + which the role belongs provided you with an external ID, then provide + that value in the ExternalId parameter. This value can be any string, + such as a passphrase or account number. A cross-account role is + usually set up to trust everyone in an account. Therefore, the administrator + of the trusting account might send an external ID to the administrator + of the trusted account. That way, only someone with the ID can assume + the role, rather than everyone in the account. For more information + about the external ID, see How to Use an External ID When Granting + Access to Your AWS Resources to a Third Party in the IAM User Guide. + type: string + inlinePolicy: + description: An IAM policy as a JSON-encoded string that you want + to use as an inline session policy. + type: string + policyARNs: + description: The Amazon Resource Names (ARNs) of the IAM managed policies + that you want to use as managed session policies. The policies must + exist in the same account as the role. + items: + type: string + type: array + roleARN: + description: The Amazon Resource Name (ARN) of the role to assume. + type: string + sessionName: + description: An identifier for the assumed role session + type: string + sourceIdentityRef: + description: SourceIdentityRef is a reference to another identity + which will be chained to do role assumption. All identity types + are accepted. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + required: + - roleARN + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSClusterRoleIdentity is the Schema for the awsclusterroleidentities + API It is used to assume a role using the provided sourceRef. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterRoleIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + durationSeconds: + description: The duration, in seconds, of the role session before + it is renewed. + format: int32 + maximum: 43200 + minimum: 900 + type: integer + externalID: + description: A unique identifier that might be required when you assume + a role in another account. If the administrator of the account to + which the role belongs provided you with an external ID, then provide + that value in the ExternalId parameter. This value can be any string, + such as a passphrase or account number. A cross-account role is + usually set up to trust everyone in an account. Therefore, the administrator + of the trusting account might send an external ID to the administrator + of the trusted account. That way, only someone with the ID can assume + the role, rather than everyone in the account. For more information + about the external ID, see How to Use an External ID When Granting + Access to Your AWS Resources to a Third Party in the IAM User Guide. + type: string + inlinePolicy: + description: An IAM policy as a JSON-encoded string that you want + to use as an inline session policy. + type: string + policyARNs: + description: The Amazon Resource Names (ARNs) of the IAM managed policies + that you want to use as managed session policies. The policies must + exist in the same account as the role. + items: + type: string + type: array + roleARN: + description: The Amazon Resource Name (ARN) of the role to assume. + type: string + sessionName: + description: An identifier for the assumed role session + type: string + sourceIdentityRef: + description: SourceIdentityRef is a reference to another identity + which will be chained to do role assumption. All identity types + are accepted. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + required: + - roleARN + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AWSClusterRoleIdentity is the Schema for the awsclusterroleidentities + API It is used to assume a role using the provided sourceRef. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterRoleIdentity. + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + durationSeconds: + description: The duration, in seconds, of the role session before + it is renewed. + format: int32 + maximum: 43200 + minimum: 900 + type: integer + externalID: + description: A unique identifier that might be required when you assume + a role in another account. If the administrator of the account to + which the role belongs provided you with an external ID, then provide + that value in the ExternalId parameter. This value can be any string, + such as a passphrase or account number. A cross-account role is + usually set up to trust everyone in an account. Therefore, the administrator + of the trusting account might send an external ID to the administrator + of the trusted account. That way, only someone with the ID can assume + the role, rather than everyone in the account. For more information + about the external ID, see How to Use an External ID When Granting + Access to Your AWS Resources to a Third Party in the IAM User Guide. + type: string + inlinePolicy: + description: An IAM policy as a JSON-encoded string that you want + to use as an inline session policy. + type: string + policyARNs: + description: The Amazon Resource Names (ARNs) of the IAM managed policies + that you want to use as managed session policies. The policies must + exist in the same account as the role. + items: + type: string + type: array + roleARN: + description: The Amazon Resource Name (ARN) of the role to assume. + type: string + sessionName: + description: An identifier for the assumed role session + type: string + sourceIdentityRef: + description: SourceIdentityRef is a reference to another identity + which will be chained to do role assumption. All identity types + are accepted. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + required: + - roleARN + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsclusters.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSCluster + listKind: AWSClusterList + plural: awsclusters + shortNames: + - awsc + singular: awscluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this AWSCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Cluster infrastructure is ready for EC2 instances + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the cluster is using + jsonPath: .spec.networkSpec.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSCluster is the Schema for the awsclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSClusterSpec defines the desired state of AWSCluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration for + customizing control plane behavior. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups sets the security groups + used by the load balancer. Expected to be security group IDs + This is optional - if not provided new security groups will + be created for the load balancer + items: + type: string + type: array + crossZoneLoadBalancing: + description: "CrossZoneLoadBalancing enables the classic ELB cross + availability zone balancing. \n With cross-zone load balancing, + each load balancer node for your Classic Load Balancer distributes + requests evenly across the registered instances in all enabled + Availability Zones. If cross-zone load balancing is disabled, + each load balancer node distributes requests evenly across the + registered instances in its Availability Zone only. \n Defaults + to false." + type: boolean + scheme: + default: internet-facing + description: Scheme sets the scheme of the load balancer (defaults + to internet-facing) + enum: + - internet-facing + - Internet-facing + - internal + type: string + subnets: + description: Subnets sets the subnets that should be applied to + the control plane load balancer (defaults to discovered subnets + for managed VPCs or an empty set for unmanaged VPCs) + items: + type: string + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling this cluster + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + networkSpec: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + region: + description: The AWS Region the cluster lives in. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + type: object + status: + description: AWSClusterStatus defines the observed state of AWSCluster. + properties: + bastion: + description: Instance describes an AWS instance. + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + required: + - id + type: object + conditions: + description: Conditions provide observations of the operational state + of a Cluster API resource. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains is a slice of FailureDomains. + type: object + network: + description: Network encapsulates AWS networking resources. + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + ready: + default: false + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Cluster infrastructure is ready for EC2 instances + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the cluster is using + jsonPath: .spec.network.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSCluster is the Schema for the awsclusters API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSClusterSpec defines the desired state of AWSCluster + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration for + customizing control plane behavior. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups sets the security groups + used by the load balancer. Expected to be security group IDs + This is optional - if not provided new security groups will + be created for the load balancer + items: + type: string + type: array + crossZoneLoadBalancing: + description: "CrossZoneLoadBalancing enables the classic ELB cross + availability zone balancing. \n With cross-zone load balancing, + each load balancer node for your Classic Load Balancer distributes + requests evenly across the registered instances in all enabled + Availability Zones. If cross-zone load balancing is disabled, + each load balancer node distributes requests evenly across the + registered instances in its Availability Zone only. \n Defaults + to false." + type: boolean + scheme: + default: internet-facing + description: Scheme sets the scheme of the load balancer (defaults + to internet-facing) + enum: + - internet-facing + - Internet-facing + - internal + type: string + subnets: + description: Subnets sets the subnets that should be applied to + the control plane load balancer (defaults to discovered subnets + for managed VPCs or an empty set for unmanaged VPCs) + items: + type: string + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling this cluster + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + network: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + region: + description: The AWS Region the cluster lives in. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + type: object + status: + description: AWSClusterStatus defines the observed state of AWSCluster + properties: + bastion: + description: Instance describes an AWS instance. + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + volumeIDs: + description: IDs of the instance's volumes + items: + type: string + type: array + required: + - id + type: object + conditions: + description: Conditions provide observations of the operational state + of a Cluster API resource. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains is a slice of FailureDomains. + type: object + networkStatus: + description: NetworkStatus encapsulates AWS networking resources. + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + ready: + default: false + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSCluster belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Cluster infrastructure is ready for EC2 instances + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the cluster is using + jsonPath: .spec.network.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSCluster is the schema for Amazon EC2 based Kubernetes Cluster + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSClusterSpec defines the desired state of an EC2-based + Kubernetes cluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration for + customizing control plane behavior. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups sets the security groups + used by the load balancer. Expected to be security group IDs + This is optional - if not provided new security groups will + be created for the load balancer + items: + type: string + type: array + crossZoneLoadBalancing: + description: "CrossZoneLoadBalancing enables the classic ELB cross + availability zone balancing. \n With cross-zone load balancing, + each load balancer node for your Classic Load Balancer distributes + requests evenly across the registered instances in all enabled + Availability Zones. If cross-zone load balancing is disabled, + each load balancer node distributes requests evenly across the + registered instances in its Availability Zone only. \n Defaults + to false." + type: boolean + healthCheckProtocol: + description: HealthCheckProtocol sets the protocol type for classic + ELB health check target default value is ClassicELBProtocolSSL + type: string + name: + description: Name sets the name of the classic ELB load balancer. + As per AWS, the name must be unique within your set of load + balancers for the region, must have a maximum of 32 characters, + must contain only alphanumeric characters or hyphens, and cannot + begin or end with a hyphen. Once set, the value cannot be changed. + maxLength: 32 + pattern: ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ + type: string + scheme: + default: internet-facing + description: Scheme sets the scheme of the load balancer (defaults + to internet-facing) + enum: + - internet-facing + - internal + type: string + subnets: + description: Subnets sets the subnets that should be applied to + the control plane load balancer (defaults to discovered subnets + for managed VPCs or an empty set for unmanaged VPCs) + items: + type: string + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling this cluster + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + network: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + partition: + description: Partition is the AWS security partition being used. Defaults + to "aws" + type: string + region: + description: The AWS Region the cluster lives in. + type: string + s3Bucket: + description: S3Bucket contains options to configure a supporting S3 + bucket for this cluster - currently used for nodes requiring Ignition + (https://coreos.github.io/ignition/) for bootstrapping (requires + BootstrapFormatIgnition feature flag to be enabled). + properties: + controlPlaneIAMInstanceProfile: + description: ControlPlaneIAMInstanceProfile is a name of the IAMInstanceProfile, + which will be allowed to read control-plane node bootstrap data + from S3 Bucket. + type: string + name: + description: Name defines name of S3 Bucket to be created. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + nodesIAMInstanceProfiles: + description: NodesIAMInstanceProfiles is a list of IAM instance + profiles, which will be allowed to read worker nodes bootstrap + data from S3 Bucket. + items: + type: string + type: array + required: + - controlPlaneIAMInstanceProfile + - name + - nodesIAMInstanceProfiles + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + type: object + status: + description: AWSClusterStatus defines the observed state of AWSCluster. + properties: + bastion: + description: Instance describes an AWS instance. + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + volumeIDs: + description: IDs of the instance's volumes + items: + type: string + type: array + required: + - id + type: object + conditions: + description: Conditions provide observations of the operational state + of a Cluster API resource. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains is a slice of FailureDomains. + type: object + networkStatus: + description: NetworkStatus encapsulates AWS networking resources. + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + ready: + default: false + type: boolean + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + clusterctl.cluster.x-k8s.io/move-hierarchy: "" + name: awsclusterstaticidentities.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSClusterStaticIdentity + listKind: AWSClusterStaticIdentityList + plural: awsclusterstaticidentities + shortNames: + - awssi + singular: awsclusterstaticidentity + scope: Cluster + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSClusterStaticIdentity is the Schema for the awsclusterstaticidentities + API It represents a reference to an AWS access key ID and secret access + key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterStaticIdentity + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + AllowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: AccessKeyID: AKIAIOSFODNN7EXAMPLE + SecretAccessKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY SessionToken: + Optional' + properties: + name: + description: Name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: Namespace defines the space within which the secret + name must be unique. + type: string + type: object + required: + - secretRef + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSClusterStaticIdentity is the Schema for the awsclusterstaticidentities + API It represents a reference to an AWS access key ID and secret access + key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterStaticIdentity + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: AccessKeyID: AKIAIOSFODNN7EXAMPLE + SecretAccessKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY SessionToken: + Optional' + type: string + required: + - secretRef + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AWSClusterStaticIdentity is the Schema for the awsclusterstaticidentities + API It represents a reference to an AWS access key ID and secret access + key, stored in a secret. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec for this AWSClusterStaticIdentity + properties: + allowedNamespaces: + description: AllowedNamespaces is used to identify which namespaces + are allowed to use the identity from. Namespaces can be selected + either using an array of namespaces or with label selector. An empty + allowedNamespaces object indicates that AWSClusters can use this + identity from any namespace. If this object is nil, no namespaces + will be allowed (default behaviour, if this field is not provided) + A namespace should be either in the NamespaceList or match with + Selector to use the identity. + nullable: true + properties: + list: + description: An nil or empty list indicates that AWSClusters cannot + use the identity from any namespace. + items: + type: string + nullable: true + type: array + selector: + description: An empty selector indicates that AWSClusters cannot + use this AWSClusterIdentity from any namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object + secretRef: + description: 'Reference to a secret containing the credentials. The + secret should contain the following data keys: AccessKeyID: AKIAIOSFODNN7EXAMPLE + SecretAccessKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY SessionToken: + Optional' + type: string + required: + - secretRef + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsclustertemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSClusterTemplate + listKind: AWSClusterTemplateList + plural: awsclustertemplates + shortNames: + - awsct + singular: awsclustertemplate + scope: Namespaced + versions: + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSClusterTemplate is the Schema for the awsclustertemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSClusterTemplateSpec defines the desired state of AWSClusterTemplate. + properties: + template: + properties: + spec: + description: AWSClusterSpec defines the desired state of AWSCluster + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to AWS resources managed by the AWS provider, in addition + to the ones added by default. + type: object + bastion: + description: Bastion contains options to configure the bastion + host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks + allowed to access the bastion host. They are set as + ingress rules for the Bastion host's Security Group + (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the + bastion. If not specified, the AMI will default to one + picked out in public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are + no Ingress rules in the bastion host's security group. + Requires AllowedCIDRBlocks to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a + bastion host instance with a public ip to access the + VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance + type for the bastion. If not specified, Cluster API + Provider AWS will use t3.micro for all regions except + us-east-1, where t2.micro will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint + used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration + for customizing control plane behavior. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups sets the security + groups used by the load balancer. Expected to be security + group IDs This is optional - if not provided new security + groups will be created for the load balancer + items: + type: string + type: array + crossZoneLoadBalancing: + description: "CrossZoneLoadBalancing enables the classic + ELB cross availability zone balancing. \n With cross-zone + load balancing, each load balancer node for your Classic + Load Balancer distributes requests evenly across the + registered instances in all enabled Availability Zones. + If cross-zone load balancing is disabled, each load + balancer node distributes requests evenly across the + registered instances in its Availability Zone only. + \n Defaults to false." + type: boolean + scheme: + default: internet-facing + description: Scheme sets the scheme of the load balancer + (defaults to internet-facing) + enum: + - internet-facing + - Internet-facing + - internal + type: string + subnets: + description: Subnets sets the subnets that should be applied + to the control plane load balancer (defaults to discovered + subnets for managed VPCs or an empty set for unmanaged + VPCs) + items: + type: string + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be + used when reconciling this cluster + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system used to look up machine images when a machine does + not specify an AMI. When set, this will be used for all + cluster machines unless a machine specifies a different + ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to + look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines + unless a machine specifies a different ImageLookupOrg. Supports + substitutions for {{.BaseOS}} and {{.K8sVersion}} with the + base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the + ubuntu base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to + look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines + unless a machine specifies a different ImageLookupOrg. + type: string + network: + description: NetworkSpec encapsulates all things related to + AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply + to control plane and worker node security groups. + The source for the rule will be set to control plane + and worker security group IDs. + items: + description: CNIIngressRule defines an AWS ingress + rule for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the + protocol type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set + of security groups to use for cluster instances This + is optional - if not provided new security groups will + be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability + zone to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used + when the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public + subnet. A subnet is public when it is associated + with a route table that has a route to an internet + gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id + associated with the subnet. Ignored unless the + subnet is managed by the provider, in which case + this is set on the public subnet where the NAT + gateway resides. It is then used to determine + routes for private subnets in the same AZ as the + public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id + associated with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies + how AZs should be selected if there are more AZs + in a region than specified by AvailabilityZoneUsageLimit. + There are 2 selection schemes: Ordered - selects + based on alphabetical order Random - selects AZs + randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies + the maximum number of availability zones (AZ) that + should be used in a region when automatically creating + subnets. If a region has more than this number of + AZs then this number of AZs will be picked randomly + when creating default subnets. Defaults to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used + when the provider creates a managed VPC. Defaults + to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider + should use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet + gateway associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: object + region: + description: The AWS Region the cluster lives in. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach + to the bastion host. Valid values are empty string (do not + use SSH keys), a valid SSH key name, or omitted (use the + default SSH key name) + type: string + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - additionalPrinterColumns: + - description: Time duration since creation of AWSClusterTemplate + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSClusterTemplate is the schema for Amazon EC2 based Kubernetes + Cluster Templates. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSClusterTemplateSpec defines the desired state of AWSClusterTemplate. + properties: + template: + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: AWSClusterSpec defines the desired state of an EC2-based + Kubernetes cluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to AWS resources managed by the AWS provider, in addition + to the ones added by default. + type: object + bastion: + description: Bastion contains options to configure the bastion + host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks + allowed to access the bastion host. They are set as + ingress rules for the Bastion host's Security Group + (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the + bastion. If not specified, the AMI will default to one + picked out in public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are + no Ingress rules in the bastion host's security group. + Requires AllowedCIDRBlocks to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a + bastion host instance with a public ip to access the + VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance + type for the bastion. If not specified, Cluster API + Provider AWS will use t3.micro for all regions except + us-east-1, where t2.micro will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint + used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + controlPlaneLoadBalancer: + description: ControlPlaneLoadBalancer is optional configuration + for customizing control plane behavior. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups sets the security + groups used by the load balancer. Expected to be security + group IDs This is optional - if not provided new security + groups will be created for the load balancer + items: + type: string + type: array + crossZoneLoadBalancing: + description: "CrossZoneLoadBalancing enables the classic + ELB cross availability zone balancing. \n With cross-zone + load balancing, each load balancer node for your Classic + Load Balancer distributes requests evenly across the + registered instances in all enabled Availability Zones. + If cross-zone load balancing is disabled, each load + balancer node distributes requests evenly across the + registered instances in its Availability Zone only. + \n Defaults to false." + type: boolean + healthCheckProtocol: + description: HealthCheckProtocol sets the protocol type + for classic ELB health check target default value is + ClassicELBProtocolSSL + type: string + name: + description: Name sets the name of the classic ELB load + balancer. As per AWS, the name must be unique within + your set of load balancers for the region, must have + a maximum of 32 characters, must contain only alphanumeric + characters or hyphens, and cannot begin or end with + a hyphen. Once set, the value cannot be changed. + maxLength: 32 + pattern: ^[A-Za-z0-9]([A-Za-z0-9]{0,31}|[-A-Za-z0-9]{0,30}[A-Za-z0-9])$ + type: string + scheme: + default: internet-facing + description: Scheme sets the scheme of the load balancer + (defaults to internet-facing) + enum: + - internet-facing + - internal + type: string + subnets: + description: Subnets sets the subnets that should be applied + to the control plane load balancer (defaults to discovered + subnets for managed VPCs or an empty set for unmanaged + VPCs) + items: + type: string + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be + used when reconciling this cluster + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system used to look up machine images when a machine does + not specify an AMI. When set, this will be used for all + cluster machines unless a machine specifies a different + ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to + look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines + unless a machine specifies a different ImageLookupOrg. Supports + substitutions for {{.BaseOS}} and {{.K8sVersion}} with the + base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the + ubuntu base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to + look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines + unless a machine specifies a different ImageLookupOrg. + type: string + network: + description: NetworkSpec encapsulates all things related to + AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply + to control plane and worker node security groups. + The source for the rule will be set to control plane + and worker security group IDs. + items: + description: CNIIngressRule defines an AWS ingress + rule for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the + protocol type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set + of security groups to use for cluster instances This + is optional - if not provided new security groups will + be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability + zone to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used + when the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public + subnet. A subnet is public when it is associated + with a route table that has a route to an internet + gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id + associated with the subnet. Ignored unless the + subnet is managed by the provider, in which case + this is set on the public subnet where the NAT + gateway resides. It is then used to determine + routes for private subnets in the same AZ as the + public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id + associated with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies + how AZs should be selected if there are more AZs + in a region than specified by AvailabilityZoneUsageLimit. + There are 2 selection schemes: Ordered - selects + based on alphabetical order Random - selects AZs + randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies + the maximum number of availability zones (AZ) that + should be used in a region when automatically creating + subnets. If a region has more than this number of + AZs then this number of AZs will be picked randomly + when creating default subnets. Defaults to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used + when the provider creates a managed VPC. Defaults + to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider + should use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet + gateway associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing + the resource. + type: object + type: object + type: object + partition: + description: Partition is the AWS security partition being + used. Defaults to "aws" + type: string + region: + description: The AWS Region the cluster lives in. + type: string + s3Bucket: + description: S3Bucket contains options to configure a supporting + S3 bucket for this cluster - currently used for nodes requiring + Ignition (https://coreos.github.io/ignition/) for bootstrapping + (requires BootstrapFormatIgnition feature flag to be enabled). + properties: + controlPlaneIAMInstanceProfile: + description: ControlPlaneIAMInstanceProfile is a name + of the IAMInstanceProfile, which will be allowed to + read control-plane node bootstrap data from S3 Bucket. + type: string + name: + description: Name defines name of S3 Bucket to be created. + maxLength: 63 + minLength: 3 + pattern: ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ + type: string + nodesIAMInstanceProfiles: + description: NodesIAMInstanceProfiles is a list of IAM + instance profiles, which will be allowed to read worker + nodes bootstrap data from S3 Bucket. + items: + type: string + type: array + required: + - controlPlaneIAMInstanceProfile + - name + - nodesIAMInstanceProfiles + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach + to the bastion host. Valid values are empty string (do not + use SSH keys), a valid SSH key name, or omitted (use the + default SSH key name) + type: string + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsfargateprofiles.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSFargateProfile + listKind: AWSFargateProfileList + plural: awsfargateprofiles + shortNames: + - awsfp + singular: awsfargateprofile + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: AWSFargateProfile ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EKS Fargate profile name + jsonPath: .spec.profileName + name: ProfileName + type: string + - description: Failure reason + jsonPath: .status.failureReason + name: FailureReason + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSFargateProfile is the Schema for the awsfargateprofiles API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FargateProfileSpec defines the desired state of FargateProfile + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + clusterName: + description: ClusterName is the name of the Cluster this object belongs + to. + minLength: 1 + type: string + profileName: + description: ProfileName specifies the profile name. + type: string + roleName: + description: RoleName specifies the name of IAM role for this fargate + pool If the role is pre-existing we will treat it as unmanaged and + not delete it on deletion. If the EKSEnableIAM feature flag is true + and no name is supplied then a role is created. + type: string + selectors: + description: Selectors specify fargate pod selectors. + items: + description: FargateSelector specifies a selector for pods that + should run on this fargate pool + properties: + labels: + additionalProperties: + type: string + description: Labels specifies which pod labels this selector + should match. + type: object + namespace: + description: Namespace specifies which namespace this selector + should match. + type: string + type: object + type: array + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup. + items: + type: string + type: array + required: + - clusterName + type: object + status: + description: FargateProfileStatus defines the observed state of FargateProfile + properties: + conditions: + description: Conditions defines current state of the Fargate profile. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the FargateProfile's spec or the configuration of the + controller, and that manual intervention is required. Examples of + terminal errors would be invalid combinations of settings in the + spec, values that are unsupported by the controller, or the responsible + controller itself being critically misconfigured. \n Any transient + errors that occur during the reconciliation of FargateProfiles can + be added as events to the FargateProfile object and/or logged in + the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the FargateProfile's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of FargateProfiles can be added + as events to the FargateProfile object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the FargateProfile is available. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AWSFargateProfile ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EKS Fargate profile name + jsonPath: .spec.profileName + name: ProfileName + type: string + - description: Failure reason + jsonPath: .status.failureReason + name: FailureReason + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSFargateProfile is the Schema for the awsfargateprofiles API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FargateProfileSpec defines the desired state of FargateProfile + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + clusterName: + description: ClusterName is the name of the Cluster this object belongs + to. + minLength: 1 + type: string + profileName: + description: ProfileName specifies the profile name. + type: string + roleName: + description: RoleName specifies the name of IAM role for this fargate + pool If the role is pre-existing we will treat it as unmanaged and + not delete it on deletion. If the EKSEnableIAM feature flag is true + and no name is supplied then a role is created. + type: string + selectors: + description: Selectors specify fargate pod selectors. + items: + description: FargateSelector specifies a selector for pods that + should run on this fargate pool + properties: + labels: + additionalProperties: + type: string + description: Labels specifies which pod labels this selector + should match. + type: object + namespace: + description: Namespace specifies which namespace this selector + should match. + type: string + type: object + type: array + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup. + items: + type: string + type: array + required: + - clusterName + type: object + status: + description: FargateProfileStatus defines the observed state of FargateProfile + properties: + conditions: + description: Conditions defines current state of the Fargate profile. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the FargateProfile's spec or the configuration of the + controller, and that manual intervention is required. Examples of + terminal errors would be invalid combinations of settings in the + spec, values that are unsupported by the controller, or the responsible + controller itself being critically misconfigured. \n Any transient + errors that occur during the reconciliation of FargateProfiles can + be added as events to the FargateProfile object and/or logged in + the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the FargateProfile's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of FargateProfiles can be added + as events to the FargateProfile object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the FargateProfile is available. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: AWSFargateProfile ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EKS Fargate profile name + jsonPath: .spec.profileName + name: ProfileName + type: string + - description: Failure reason + jsonPath: .status.failureReason + name: FailureReason + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSFargateProfile is the Schema for the awsfargateprofiles API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FargateProfileSpec defines the desired state of FargateProfile. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + clusterName: + description: ClusterName is the name of the Cluster this object belongs + to. + minLength: 1 + type: string + profileName: + description: ProfileName specifies the profile name. + type: string + roleName: + description: RoleName specifies the name of IAM role for this fargate + pool If the role is pre-existing we will treat it as unmanaged and + not delete it on deletion. If the EKSEnableIAM feature flag is true + and no name is supplied then a role is created. + type: string + selectors: + description: Selectors specify fargate pod selectors. + items: + description: FargateSelector specifies a selector for pods that + should run on this fargate pool. + properties: + labels: + additionalProperties: + type: string + description: Labels specifies which pod labels this selector + should match. + type: object + namespace: + description: Namespace specifies which namespace this selector + should match. + type: string + type: object + type: array + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup. + items: + type: string + type: array + required: + - clusterName + type: object + status: + description: FargateProfileStatus defines the observed state of FargateProfile. + properties: + conditions: + description: Conditions defines current state of the Fargate profile. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the FargateProfile's spec or the configuration of the + controller, and that manual intervention is required. Examples of + terminal errors would be invalid combinations of settings in the + spec, values that are unsupported by the controller, or the responsible + controller itself being critically misconfigured. \n Any transient + errors that occur during the reconciliation of FargateProfiles can + be added as events to the FargateProfile object and/or logged in + the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the FargateProfile and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the FargateProfile's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of FargateProfiles can be added + as events to the FargateProfile object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the FargateProfile is available. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsmachinepools.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSMachinePool + listKind: AWSMachinePoolList + plural: awsmachinepools + shortNames: + - awsmp + singular: awsmachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Machine ready status + jsonPath: .status.replicas + name: Replicas + type: integer + - description: Minimum instanes in ASG + jsonPath: .spec.minSize + name: MinSize + type: integer + - description: Maximum instanes in ASG + jsonPath: .spec.maxSize + name: MaxSize + type: integer + - description: Launch Template ID + jsonPath: .status.launchTemplateID + name: LaunchTemplate ID + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSMachinePool is the Schema for the awsmachinepools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachinePoolSpec defines the desired state of AWSMachinePool + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + type: object + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + awsLaunchTemplate: + description: AWSLaunchTemplate specifies the launch template and version + to use when an instance is launched. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instances. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID, ARN, or filters. Only one of ID, ARN or + Filters may be specified. Specifying more than one will result + in a validation error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the + rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the rules + defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + iamInstanceProfile: + description: The name or the Amazon Resource Name (ARN) of the + instance profile associated with the IAM role for the instance. + The instance profile contains the IAM role. + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look + up the image for this machine It will be ignored if an explicit + AMI is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use + for image lookup if AMI is not set. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + name: + description: The name of the launch template. + type: string + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to + the instance. Valid values are empty string (do not use SSH + keys), a valid SSH key name, or omitted (use the default SSH + key name) + type: string + versionNumber: + description: 'VersionNumber is the version of the launch template + that is applied. Typically a new version is created when at + least one of the following happens: 1) A new launch template + spec is applied. 2) One or more parameters in an existing template + is changed. 3) A new AMI is discovered.' + format: int64 + type: integer + type: object + capacityRebalance: + description: Enable or disable the capacity rebalance autoscaling + group feature + type: boolean + defaultCoolDown: + description: The amount of time, in seconds, after a scaling activity + completes before another scaling activity can start. If no value + is supplied by user a default value of 300 seconds is set + type: string + maxSize: + default: 1 + description: MaxSize defines the maximum size of the group. + format: int32 + minimum: 1 + type: integer + minSize: + default: 1 + description: MinSize defines the minimum size of the group. + format: int32 + minimum: 1 + type: integer + mixedInstancesPolicy: + description: MixedInstancesPolicy describes how multiple instance + types will be used by the ASG. + properties: + instancesDistribution: + description: InstancesDistribution to configure distribution of + On-Demand Instances and Spot Instances. + properties: + onDemandAllocationStrategy: + default: prioritized + description: OnDemandAllocationStrategy indicates how to allocate + instance types to fulfill On-Demand capacity. + enum: + - prioritized + type: string + onDemandBaseCapacity: + default: 0 + format: int64 + type: integer + onDemandPercentageAboveBaseCapacity: + default: 100 + format: int64 + type: integer + spotAllocationStrategy: + default: lowest-price + description: SpotAllocationStrategy indicates how to allocate + instances across Spot Instance pools. + enum: + - lowest-price + - capacity-optimized + type: string + type: object + overrides: + items: + description: Overrides are used to override the instance type + specified by the launch template with multiple instance types + that can be used to launch On-Demand Instances and Spot Instances. + properties: + instanceType: + type: string + required: + - instanceType + type: object + type: array + type: object + providerID: + description: ProviderID is the ARN of the associated ASG + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + refreshPreferences: + description: RefreshPreferences describes set of preferences associated + with the instance refresh request. + properties: + instanceWarmup: + description: The number of seconds until a newly launched instance + is configured and ready to use. During this time, the next replacement + will not be initiated. The default is to use the value for the + health check grace period defined for the group. + format: int64 + type: integer + minHealthyPercentage: + description: The amount of capacity as a percentage in ASG that + must remain healthy during an instance refresh. The default + is 90. + format: int64 + type: integer + strategy: + description: The strategy to use for the instance refresh. The + only valid value is Rolling. A rolling update is an update that + is applied to all instances in an Auto Scaling group until all + instances have been updated. + type: string + type: object + subnets: + description: Subnets is an array of subnet configurations + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID, ARN, or filters. Only one of ID, ARN or Filters + may be specified. Specifying more than one will result in a validation + error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + required: + - awsLaunchTemplate + - maxSize + - minSize + type: object + status: + description: AWSMachinePoolStatus defines the observed state of AWSMachinePool + properties: + asgStatus: + description: ASGStatus is a status string returned by the autoscaling + API + type: string + conditions: + description: Conditions defines current service state of the AWSMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instances: + description: Instances contains the status for each instance in the + pool + items: + description: AWSMachinePoolInstanceStatus defines the status of + the AWSMachinePoolInstance. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within ASG + type: string + version: + description: Version defines the Kubernetes version for the + Machine Instance + type: string + type: object + type: array + launchTemplateID: + description: The ID of the launch template + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Machine ready status + jsonPath: .status.replicas + name: Replicas + type: integer + - description: Minimum instanes in ASG + jsonPath: .spec.minSize + name: MinSize + type: integer + - description: Maximum instanes in ASG + jsonPath: .spec.maxSize + name: MaxSize + type: integer + - description: Launch Template ID + jsonPath: .status.launchTemplateID + name: LaunchTemplate ID + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSMachinePool is the Schema for the awsmachinepools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachinePoolSpec defines the desired state of AWSMachinePool + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + type: object + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + awsLaunchTemplate: + description: AWSLaunchTemplate specifies the launch template and version + to use when an instance is launched. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instances. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID, ARN, or filters. Only one of ID, ARN or + Filters may be specified. Specifying more than one will result + in a validation error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the + rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look + up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + iamInstanceProfile: + description: The name or the Amazon Resource Name (ARN) of the + instance profile associated with the IAM role for the instance. + The instance profile contains the IAM role. + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look + up the image for this machine It will be ignored if an explicit + AMI is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use + for image lookup if AMI is not set. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + name: + description: The name of the launch template. + type: string + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to + the instance. Valid values are empty string (do not use SSH + keys), a valid SSH key name, or omitted (use the default SSH + key name) + type: string + versionNumber: + description: 'VersionNumber is the version of the launch template + that is applied. Typically a new version is created when at + least one of the following happens: 1) A new launch template + spec is applied. 2) One or more parameters in an existing template + is changed. 3) A new AMI is discovered.' + format: int64 + type: integer + type: object + capacityRebalance: + description: Enable or disable the capacity rebalance autoscaling + group feature + type: boolean + defaultCoolDown: + description: The amount of time, in seconds, after a scaling activity + completes before another scaling activity can start. If no value + is supplied by user a default value of 300 seconds is set + type: string + maxSize: + default: 1 + description: MaxSize defines the maximum size of the group. + format: int32 + minimum: 1 + type: integer + minSize: + default: 1 + description: MinSize defines the minimum size of the group. + format: int32 + minimum: 1 + type: integer + mixedInstancesPolicy: + description: MixedInstancesPolicy describes how multiple instance + types will be used by the ASG. + properties: + instancesDistribution: + description: InstancesDistribution to configure distribution of + On-Demand Instances and Spot Instances. + properties: + onDemandAllocationStrategy: + default: prioritized + description: OnDemandAllocationStrategy indicates how to allocate + instance types to fulfill On-Demand capacity. + enum: + - prioritized + type: string + onDemandBaseCapacity: + default: 0 + format: int64 + type: integer + onDemandPercentageAboveBaseCapacity: + default: 100 + format: int64 + type: integer + spotAllocationStrategy: + default: lowest-price + description: SpotAllocationStrategy indicates how to allocate + instances across Spot Instance pools. + enum: + - lowest-price + - capacity-optimized + type: string + type: object + overrides: + items: + description: Overrides are used to override the instance type + specified by the launch template with multiple instance types + that can be used to launch On-Demand Instances and Spot Instances. + properties: + instanceType: + type: string + required: + - instanceType + type: object + type: array + type: object + providerID: + description: ProviderID is the ARN of the associated ASG + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + refreshPreferences: + description: RefreshPreferences describes set of preferences associated + with the instance refresh request. + properties: + instanceWarmup: + description: The number of seconds until a newly launched instance + is configured and ready to use. During this time, the next replacement + will not be initiated. The default is to use the value for the + health check grace period defined for the group. + format: int64 + type: integer + minHealthyPercentage: + description: The amount of capacity as a percentage in ASG that + must remain healthy during an instance refresh. The default + is 90. + format: int64 + type: integer + strategy: + description: The strategy to use for the instance refresh. The + only valid value is Rolling. A rolling update is an update that + is applied to all instances in an Auto Scaling group until all + instances have been updated. + type: string + type: object + subnets: + description: Subnets is an array of subnet configurations + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID, ARN, or filters. Only one of ID, ARN or Filters + may be specified. Specifying more than one will result in a validation + error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + required: + - awsLaunchTemplate + - maxSize + - minSize + type: object + status: + description: AWSMachinePoolStatus defines the observed state of AWSMachinePool + properties: + asgStatus: + description: ASGStatus is a status string returned by the autoscaling + API + type: string + conditions: + description: Conditions defines current service state of the AWSMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instances: + description: Instances contains the status for each instance in the + pool + items: + description: AWSMachinePoolInstanceStatus defines the status of + the AWSMachinePoolInstance. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within ASG + type: string + version: + description: Version defines the Kubernetes version for the + Machine Instance + type: string + type: object + type: array + launchTemplateID: + description: The ID of the launch template + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Machine ready status + jsonPath: .status.replicas + name: Replicas + type: integer + - description: Minimum instanes in ASG + jsonPath: .spec.minSize + name: MinSize + type: integer + - description: Maximum instanes in ASG + jsonPath: .spec.maxSize + name: MaxSize + type: integer + - description: Launch Template ID + jsonPath: .status.launchTemplateID + name: LaunchTemplate ID + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSMachinePool is the Schema for the awsmachinepools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachinePoolSpec defines the desired state of AWSMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + type: object + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + awsLaunchTemplate: + description: AWSLaunchTemplate specifies the launch template and version + to use when an instance is launched. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instances. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID or filters. Only one of ID or Filters may + be specified. Specifying more than one will result in a validation + error. + properties: + arn: + description: 'ARN of resource. Deprecated: This field has + no function and is going to be removed in the next release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the + rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource. + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look + up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + iamInstanceProfile: + description: The name or the Amazon Resource Name (ARN) of the + instance profile associated with the IAM role for the instance. + The instance profile contains the IAM role. + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look + up the image for this machine It will be ignored if an explicit + AMI is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use + for image lookup if AMI is not set. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + name: + description: The name of the launch template. + type: string + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to + the instance. Valid values are empty string (do not use SSH + keys), a valid SSH key name, or omitted (use the default SSH + key name) + type: string + versionNumber: + description: 'VersionNumber is the version of the launch template + that is applied. Typically a new version is created when at + least one of the following happens: 1) A new launch template + spec is applied. 2) One or more parameters in an existing template + is changed. 3) A new AMI is discovered.' + format: int64 + type: integer + type: object + capacityRebalance: + description: Enable or disable the capacity rebalance autoscaling + group feature + type: boolean + defaultCoolDown: + description: The amount of time, in seconds, after a scaling activity + completes before another scaling activity can start. If no value + is supplied by user a default value of 300 seconds is set + type: string + maxSize: + default: 1 + description: MaxSize defines the maximum size of the group. + format: int32 + minimum: 1 + type: integer + minSize: + default: 1 + description: MinSize defines the minimum size of the group. + format: int32 + minimum: 0 + type: integer + mixedInstancesPolicy: + description: MixedInstancesPolicy describes how multiple instance + types will be used by the ASG. + properties: + instancesDistribution: + description: InstancesDistribution to configure distribution of + On-Demand Instances and Spot Instances. + properties: + onDemandAllocationStrategy: + default: prioritized + description: OnDemandAllocationStrategy indicates how to allocate + instance types to fulfill On-Demand capacity. + enum: + - prioritized + type: string + onDemandBaseCapacity: + default: 0 + format: int64 + type: integer + onDemandPercentageAboveBaseCapacity: + default: 100 + format: int64 + type: integer + spotAllocationStrategy: + default: lowest-price + description: SpotAllocationStrategy indicates how to allocate + instances across Spot Instance pools. + enum: + - lowest-price + - capacity-optimized + type: string + type: object + overrides: + items: + description: Overrides are used to override the instance type + specified by the launch template with multiple instance types + that can be used to launch On-Demand Instances and Spot Instances. + properties: + instanceType: + type: string + required: + - instanceType + type: object + type: array + type: object + providerID: + description: ProviderID is the ARN of the associated ASG + type: string + providerIDList: + description: ProviderIDList are the identification IDs of machine + instances provided by the provider. This field must match the provider + IDs as seen on the node objects corresponding to a machine pool's + machine instances. + items: + type: string + type: array + refreshPreferences: + description: RefreshPreferences describes set of preferences associated + with the instance refresh request. + properties: + instanceWarmup: + description: The number of seconds until a newly launched instance + is configured and ready to use. During this time, the next replacement + will not be initiated. The default is to use the value for the + health check grace period defined for the group. + format: int64 + type: integer + minHealthyPercentage: + description: The amount of capacity as a percentage in ASG that + must remain healthy during an instance refresh. The default + is 90. + format: int64 + type: integer + strategy: + description: The strategy to use for the instance refresh. The + only valid value is Rolling. A rolling update is an update that + is applied to all instances in an Auto Scaling group until all + instances have been updated. + type: string + type: object + subnets: + description: Subnets is an array of subnet configurations + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID or filters. Only one of ID or Filters may be specified. + Specifying more than one will result in a validation error. + properties: + arn: + description: 'ARN of resource. Deprecated: This field has no + function and is going to be removed in the next release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource. + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + required: + - awsLaunchTemplate + - maxSize + - minSize + type: object + status: + description: AWSMachinePoolStatus defines the observed state of AWSMachinePool. + properties: + asgStatus: + description: ASGStatus is a status string returned by the autoscaling + API. + type: string + conditions: + description: Conditions defines current service state of the AWSMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instances: + description: Instances contains the status for each instance in the + pool + items: + description: AWSMachinePoolInstanceStatus defines the status of + the AWSMachinePoolInstance. + properties: + instanceID: + description: InstanceID is the identification of the Machine + Instance within ASG + type: string + version: + description: Version defines the Kubernetes version for the + Machine Instance + type: string + type: object + type: array + launchTemplateID: + description: The ID of the launch template + type: string + ready: + description: Ready is true when the provider resource is ready. + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsmachines.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSMachine + listKind: AWSMachineList + plural: awsmachines + shortNames: + - awsm + singular: awsmachine + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this AWSMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: EC2 instance state + jsonPath: .status.instanceState + name: State + type: string + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EC2 instance ID + jsonPath: .spec.providerID + name: InstanceID + type: string + - description: Machine object which owns with this AWSMachine + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSMachine is the Schema for the awsmachines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineSpec defines the desired state of AWSMachine + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references to + security groups that should be applied to the instance. These security + groups would be set in addition to any security groups defined at + the cluster level or in the actuator. It is possible to specify + either IDs of Filters. Using Filters will cause additional requests + to AWS API and if tags change the attached security groups might + change too. + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID, ARN, or filters. Only one of ID, ARN or Filters + may be specified. Specifying more than one will result in a validation + error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + If both the AWSCluster and the AWSMachine specify the same tag name + with different values, the AWSMachine's value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined by + the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true will + not use AWS Secrets Manager or AWS Systems Manager Parameter + Store to ensure privacy of userdata. By default, a cloud-init + boothook shell script is prepended to download the userdata + from Secrets Manager and additionally delete the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used to form + the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret name. This + is stored temporarily, and deleted when the machine registers + as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage to distribute + secrets. By default or with the value of secrets-manager, will + use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. For + this infrastructure provider, the ID is equivalent to an AWS Availability + Zone. If multiple subnets are matched for the availability zone, + the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance profile + to assign to the instance + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + the image for this machine It will be ignored if an explicit AMI + is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced by + kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use for + image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. Example: + m4.xlarge' + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate with + the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for the + storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be used. + The key must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should get a + public IP. Precedence for this setting is as follows: 1. This field + if set 2. Cluster/flavor setting 3. Subnet default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options for + the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt the + volume. Can be either a KMS key ID or ARN. If Encrypted is set + and this is omitted, the default AWS key will be used. The key + must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + instance. Valid values are empty string (do not use SSH keys), a + valid SSH key name, or omitted (use the default SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for this instance. + If not specified, the cluster subnet will be used. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined by + the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared or + single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user data is + gzip-compressed before it is sent to ec2 instance. cloud-init has + built-in support for gzip-compressed user data user data stored + in aws secret manager is always gzip-compressed. + type: boolean + type: object + status: + description: AWSMachineStatus defines the observed state of AWSMachine + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AWSMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instanceState: + description: InstanceState is the state of the AWS instance for this + machine. + type: string + interruptible: + description: Interruptible reports that this machine is using spot + instances and can therefore be interrupted by CAPI when it receives + a notice that the spot instance is to be terminated by AWS. This + will be set to true when SpotMarketOptions is not nil (i.e. this + machine is using a spot instance). + type: boolean + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: EC2 instance state + jsonPath: .status.instanceState + name: State + type: string + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EC2 instance ID + jsonPath: .spec.providerID + name: InstanceID + type: string + - description: Machine object which owns with this AWSMachine + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSMachine is the Schema for the awsmachines API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineSpec defines the desired state of AWSMachine + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references to + security groups that should be applied to the instance. These security + groups would be set in addition to any security groups defined at + the cluster level or in the actuator. It is possible to specify + either IDs of Filters. Using Filters will cause additional requests + to AWS API and if tags change the attached security groups might + change too. + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID, ARN, or filters. Only one of ID, ARN or Filters + may be specified. Specifying more than one will result in a validation + error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + If both the AWSCluster and the AWSMachine specify the same tag name + with different values, the AWSMachine's value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look up + an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true will + not use AWS Secrets Manager or AWS Systems Manager Parameter + Store to ensure privacy of userdata. By default, a cloud-init + boothook shell script is prepended to download the userdata + from Secrets Manager and additionally delete the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used to form + the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret name. This + is stored temporarily, and deleted when the machine registers + as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage to distribute + secrets. By default or with the value of secrets-manager, will + use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. For + this infrastructure provider, the ID is equivalent to an AWS Availability + Zone. If multiple subnets are matched for the availability zone, + the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance profile + to assign to the instance + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + the image for this machine It will be ignored if an explicit AMI + is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced by + kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use for + image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. Example: + m4.xlarge' + minLength: 2 + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate with + the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for the + storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be used. + The key must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should get a + public IP. Precedence for this setting is as follows: 1. This field + if set 2. Cluster/flavor setting 3. Subnet default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options for + the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt the + volume. Can be either a KMS key ID or ARN. If Encrypted is set + and this is omitted, the default AWS key will be used. The key + must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for the + volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + instance. Valid values are empty string (do not use SSH keys), a + valid SSH key name, or omitted (use the default SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for this instance. + If not specified, the cluster subnet will be used. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined by + the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared or + single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user data is + gzip-compressed before it is sent to ec2 instance. cloud-init has + built-in support for gzip-compressed user data user data stored + in aws secret manager is always gzip-compressed. + type: boolean + required: + - instanceType + type: object + status: + description: AWSMachineStatus defines the observed state of AWSMachine + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AWSMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instanceState: + description: InstanceState is the state of the AWS instance for this + machine. + type: string + interruptible: + description: Interruptible reports that this machine is using spot + instances and can therefore be interrupted by CAPI when it receives + a notice that the spot instance is to be terminated by AWS. This + will be set to true when SpotMarketOptions is not nil (i.e. this + machine is using a spot instance). + type: boolean + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSMachine belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: EC2 instance state + jsonPath: .status.instanceState + name: State + type: string + - description: Machine ready status + jsonPath: .status.ready + name: Ready + type: string + - description: EC2 instance ID + jsonPath: .spec.providerID + name: InstanceID + type: string + - description: Machine object which owns with this AWSMachine + jsonPath: .metadata.ownerReferences[?(@.kind=="Machine")].name + name: Machine + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSMachine is the schema for Amazon EC2 machines. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineSpec defines the desired state of an Amazon EC2 + instance. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references to + security groups that should be applied to the instance. These security + groups would be set in addition to any security groups defined at + the cluster level or in the actuator. It is possible to specify + either IDs of Filters. Using Filters will cause additional requests + to AWS API and if tags change the attached security groups might + change too. + items: + description: AWSResourceReference is a reference to a specific AWS + resource by ID or filters. Only one of ID or Filters may be specified. + Specifying more than one will result in a validation error. + properties: + arn: + description: 'ARN of resource. Deprecated: This field has no + function and is going to be removed in the next release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined + by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource. + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to an + instance, in addition to the ones added by default by the AWS provider. + If both the AWSCluster and the AWSMachine specify the same tag name + with different values, the AWSMachine's value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look up + an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true will + not use AWS Secrets Manager or AWS Systems Manager Parameter + Store to ensure privacy of userdata. By default, a cloud-init + boothook shell script is prepended to download the userdata + from Secrets Manager and additionally delete the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used to form + the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret name. This + is stored temporarily, and deleted when the machine registers + as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage to distribute + secrets. By default or with the value of secrets-manager, will + use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster API. For + this infrastructure provider, the ID is equivalent to an AWS Availability + Zone. If multiple subnets are matched for the availability zone, + the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance profile + to assign to the instance + type: string + ignition: + description: Ignition defined options related to the bootstrapping + systems where Ignition is used. + properties: + version: + default: "2.3" + description: Version defines which version of Ignition will be + used to generate bootstrap data. + enum: + - "2.3" + type: string + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + the image for this machine It will be ignored if an explicit AMI + is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced by + kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use for + image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. Example: + m4.xlarge' + minLength: 2 + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate with + the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for the + storage device. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be used. + The key must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified by the + cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should get a + public IP. Precedence for this setting is as follows: 1. This field + if set 2. Cluster/flavor setting 3. Subnet default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options for + the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt the + volume. Can be either a KMS key ID or ARN. If Encrypted is set + and this is omitted, the default AWS key will be used. The key + must already exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the disk. + Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for the + volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is willing + to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + instance. Valid values are empty string (do not use SSH keys), a + valid SSH key name, or omitted (use the default SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for this instance. + If not specified, the cluster subnet will be used. + properties: + arn: + description: 'ARN of resource. Deprecated: This field has no function + and is going to be removed in the next release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used to identify + a resource They are applied according to the rules defined by + the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS resource. + properties: + name: + description: Name of the filter. Filter names are case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared or + single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user data is + gzip-compressed before it is sent to ec2 instance. cloud-init has + built-in support for gzip-compressed user data user data stored + in aws secret manager is always gzip-compressed. + type: boolean + required: + - instanceType + type: object + status: + description: AWSMachineStatus defines the observed state of AWSMachine. + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + conditions: + description: Conditions defines current service state of the AWSMachine. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the Machine and will contain a more + verbose string suitable for logging and human consumption. \n This + field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of Machines can be added as events + to the Machine object and/or logged in the controller's output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the Machine and will contain a succinct + value suitable for machine interpretation. \n This field should + not be set for transitive errors that a controller faces that are + expected to be fixed automatically over time (like service outages), + but instead indicate that something is fundamentally wrong with + the Machine's spec or the configuration of the controller, and that + manual intervention is required. Examples of terminal errors would + be invalid combinations of settings in the spec, values that are + unsupported by the controller, or the responsible controller itself + being critically misconfigured. \n Any transient errors that occur + during the reconciliation of Machines can be added as events to + the Machine object and/or logged in the controller's output." + type: string + instanceState: + description: InstanceState is the state of the AWS instance for this + machine. + type: string + interruptible: + description: Interruptible reports that this machine is using spot + instances and can therefore be interrupted by CAPI when it receives + a notice that the spot instance is to be terminated by AWS. This + will be set to true when SpotMarketOptions is not nil (i.e. this + machine is using a spot instance). + type: boolean + ready: + description: Ready is true when the provider resource is ready. + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsmachinetemplates.infrastructure.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSMachineTemplate + listKind: AWSMachineTemplateList + plural: awsmachinetemplates + shortNames: + - awsmt + singular: awsmachinetemplate + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSMachineTemplate is the Schema for the awsmachinetemplates + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineTemplateSpec defines the desired state of AWSMachineTemplate + properties: + template: + description: AWSMachineTemplateResource describes the data needed + to create am AWSMachine from a template + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instance. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + It is possible to specify either IDs of Filters. Using Filters + will cause additional requests to AWS API and if tags change + the attached security groups might change too. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID, ARN, or filters. Only one of ID, ARN + or Filters may be specified. Specifying more than one + will result in a validation error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according + to the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource + properties: + name: + description: Name of the filter. Filter names + are case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the AWS provider. If both the AWSCluster and the AWSMachine + specify the same tag name with different values, the AWSMachine's + value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to + create the machine instance. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according to + the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true + will not use AWS Secrets Manager or AWS Systems Manager + Parameter Store to ensure privacy of userdata. By default, + a cloud-init boothook shell script is prepended to download + the userdata from Secrets Manager and additionally delete + the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used + to form the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret + name. This is stored temporarily, and deleted when the + machine registers as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage + to distribute secrets. By default or with the value + of secrets-manager, will use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. For this infrastructure provider, the ID is equivalent + to an AWS Availability Zone. If multiple subnets are matched + for the availability zone, the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance + profile to assign to the instance + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to + look up the image for this machine It will be ignored if + an explicit AMI is set. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, + respectively. The BaseOS will be the value in ImageLookupBaseOS + or ubuntu (the default), and the kubernetes version as defined + by the packages produced by kubernetes/release without v + as a prefix: 1.13.0, 1.12.5-mybuild.1, or 1.17.3. For example, + the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the + ubuntu base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to + use for image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate + with the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage + volumes. + items: + description: Volume encapsulates the configuration options + for the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should + be encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to + encrypt the volume. Can be either a KMS key ID or + ARN. If Encrypted is set and this is omitted, the + default AWS key will be used. The key must already + exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should + get a public IP. Precedence for this setting is as follows: + 1. This field if set 2. Cluster/flavor setting 3. Subnet + default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be + encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will + be used. The key must already exist and be accessible + by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach + to the instance. Valid values are empty string (do not use + SSH keys), a valid SSH key name, or omitted (use the default + SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for + this instance. If not specified, the cluster subnet will + be used. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according to + the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user + data is gzip-compressed before it is sent to ec2 instance. + cloud-init has built-in support for gzip-compressed user + data user data stored in aws secret manager is always gzip-compressed. + type: boolean + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSMachineTemplate is the Schema for the awsmachinetemplates + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineTemplateSpec defines the desired state of AWSMachineTemplate + properties: + template: + description: AWSMachineTemplateResource describes the data needed + to create am AWSMachine from a template + properties: + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instance. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + It is possible to specify either IDs of Filters. Using Filters + will cause additional requests to AWS API and if tags change + the attached security groups might change too. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID, ARN, or filters. Only one of ID, ARN + or Filters may be specified. Specifying more than one + will result in a validation error. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according + to the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource + properties: + name: + description: Name of the filter. Filter names + are case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the AWS provider. If both the AWSCluster and the AWSMachine + specify the same tag name with different values, the AWSMachine's + value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to + create the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will + look up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true + will not use AWS Secrets Manager or AWS Systems Manager + Parameter Store to ensure privacy of userdata. By default, + a cloud-init boothook shell script is prepended to download + the userdata from Secrets Manager and additionally delete + the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used + to form the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret + name. This is stored temporarily, and deleted when the + machine registers as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage + to distribute secrets. By default or with the value + of secrets-manager, will use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. For this infrastructure provider, the ID is equivalent + to an AWS Availability Zone. If multiple subnets are matched + for the availability zone, the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance + profile to assign to the instance + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to + look up the image for this machine It will be ignored if + an explicit AMI is set. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, + respectively. The BaseOS will be the value in ImageLookupBaseOS + or ubuntu (the default), and the kubernetes version as defined + by the packages produced by kubernetes/release without v + as a prefix: 1.13.0, 1.12.5-mybuild.1, or 1.17.3. For example, + the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the + ubuntu base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to + use for image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + minLength: 2 + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate + with the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage + volumes. + items: + description: Volume encapsulates the configuration options + for the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should + be encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to + encrypt the volume. Can be either a KMS key ID or + ARN. If Encrypted is set and this is omitted, the + default AWS key will be used. The key must already + exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should + get a public IP. Precedence for this setting is as follows: + 1. This field if set 2. Cluster/flavor setting 3. Subnet + default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be + encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will + be used. The key must already exist and be accessible + by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach + to the instance. Valid values are empty string (do not use + SSH keys), a valid SSH key name, or omitted (use the default + SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for + this instance. If not specified, the cluster subnet will + be used. + properties: + arn: + description: ARN of resource + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according to + the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user + data is gzip-compressed before it is sent to ec2 instance. + cloud-init has built-in support for gzip-compressed user + data user data stored in aws secret manager is always gzip-compressed. + type: boolean + required: + - instanceType + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: AWSMachineTemplate is the schema for the Amazon EC2 Machine Templates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSMachineTemplateSpec defines the desired state of AWSMachineTemplate. + properties: + template: + description: AWSMachineTemplateResource describes the data needed + to create am AWSMachine from a template. + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: Spec is the specification of the desired behavior + of the machine. + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instance. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + It is possible to specify either IDs of Filters. Using Filters + will cause additional requests to AWS API and if tags change + the attached security groups might change too. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID or filters. Only one of ID or Filters + may be specified. Specifying more than one will result + in a validation error. + properties: + arn: + description: 'ARN of resource. Deprecated: This field + has no function and is going to be removed in the + next release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according + to the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource. + properties: + name: + description: Name of the filter. Filter names + are case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to + add to an instance, in addition to the ones added by default + by the AWS provider. If both the AWSCluster and the AWSMachine + specify the same tag name with different values, the AWSMachine's + value takes precedence. + type: object + ami: + description: AMI is the reference to the AMI from which to + create the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will + look up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + cloudInit: + description: CloudInit defines options related to the bootstrapping + systems where CloudInit is used. + properties: + insecureSkipSecretsManager: + description: InsecureSkipSecretsManager, when set to true + will not use AWS Secrets Manager or AWS Systems Manager + Parameter Store to ensure privacy of userdata. By default, + a cloud-init boothook shell script is prepended to download + the userdata from Secrets Manager and additionally delete + the secret. + type: boolean + secretCount: + description: SecretCount is the number of secrets used + to form the complete secret + format: int32 + type: integer + secretPrefix: + description: SecretPrefix is the prefix for the secret + name. This is stored temporarily, and deleted when the + machine registers as a node against the workload cluster. + type: string + secureSecretsBackend: + description: SecureSecretsBackend, when set to parameter-store + will utilize the AWS Systems Manager Parameter Storage + to distribute secrets. By default or with the value + of secrets-manager, will use AWS Secrets Manager instead. + enum: + - secrets-manager + - ssm-parameter-store + type: string + type: object + failureDomain: + description: FailureDomain is the failure domain unique identifier + this Machine should be attached to, as defined in Cluster + API. For this infrastructure provider, the ID is equivalent + to an AWS Availability Zone. If multiple subnets are matched + for the availability zone, the first one returned is picked. + type: string + iamInstanceProfile: + description: IAMInstanceProfile is a name of an IAM instance + profile to assign to the instance + type: string + ignition: + description: Ignition defined options related to the bootstrapping + systems where Ignition is used. + properties: + version: + default: "2.3" + description: Version defines which version of Ignition + will be used to generate bootstrap data. + enum: + - "2.3" + type: string + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to + look up the image for this machine It will be ignored if + an explicit AMI is set. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, + respectively. The BaseOS will be the value in ImageLookupBaseOS + or ubuntu (the default), and the kubernetes version as defined + by the packages produced by kubernetes/release without v + as a prefix: 1.13.0, 1.12.5-mybuild.1, or 1.17.3. For example, + the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the + ubuntu base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to + use for image lookup if AMI is not set. + type: string + instanceID: + description: InstanceID is the EC2 instance ID for this machine. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + minLength: 2 + type: string + networkInterfaces: + description: NetworkInterfaces is a list of ENIs to associate + with the instance. A maximum of 2 may be specified. + items: + type: string + maxItems: 2 + type: array + nonRootVolumes: + description: Configuration options for the non root storage + volumes. + items: + description: Volume encapsulates the configuration options + for the storage device. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should + be encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to + encrypt the volume. Can be either a KMS key ID or + ARN. If Encrypted is set and this is omitted, the + default AWS key will be used. The key must already + exist and be accessible by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + type: array + providerID: + description: ProviderID is the unique identifier as specified + by the cloud provider. + type: string + publicIP: + description: 'PublicIP specifies whether the instance should + get a public IP. Precedence for this setting is as follows: + 1. This field if set 2. Cluster/flavor setting 3. Subnet + default' + type: boolean + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be + encrypted or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will + be used. The key must already exist and be accessible + by the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for + the disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size + or 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, + io1, etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions allows users to configure instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user + is willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach + to the instance. Valid values are empty string (do not use + SSH keys), a valid SSH key name, or omitted (use the default + SSH key name) + type: string + subnet: + description: Subnet is a reference to the subnet to use for + this instance. If not specified, the cluster subnet will + be used. + properties: + arn: + description: 'ARN of resource. Deprecated: This field + has no function and is going to be removed in the next + release.' + type: string + filters: + description: 'Filters is a set of key/value pairs used + to identify a resource They are applied according to + the rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an + AWS resource. + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter + values. Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + enum: + - default + - dedicated + - host + type: string + uncompressedUserData: + description: UncompressedUserData specify whether the user + data is gzip-compressed before it is sent to ec2 instance. + cloud-init has built-in support for gzip-compressed user + data user data stored in aws secret manager is always gzip-compressed. + type: boolean + required: + - instanceType + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsmanagedcontrolplanes.controlplane.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: controlplane.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSManagedControlPlane + listKind: AWSManagedControlPlaneList + plural: awsmanagedcontrolplanes + shortNames: + - awsmcp + singular: awsmanagedcontrolplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Cluster to which this AWSManagedControl belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Control plane infrastructure is ready for worker nodes + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the control plane is using + jsonPath: .spec.networkSpec.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSManagedControlPlane is the Schema for the awsmanagedcontrolplanes + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedControlPlaneSpec defines the desired state of AWSManagedControlPlane + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + addons: + description: Addons defines the EKS addons to enable with the EKS + cluster. + items: + description: Addon represents a EKS addon + properties: + conflictResolution: + default: none + description: ConflictResolution is used to declare what should + happen if there are parameter conflicts. Defaults to none + enum: + - overwrite + - none + type: string + name: + description: Name is the name of the addon + minLength: 2 + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of an IAM role + to bind to the addons service account + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - name + - version + type: object + type: array + associateOIDCProvider: + default: false + description: AssociateOIDCProvider can be enabled to automatically + create an identity provider for the controller for use with IAM + roles for service accounts + type: boolean + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + disableVPCCNI: + default: false + description: DisableVPCCNI indicates that the Amazon VPC CNI should + be disabled. With EKS clusters the Amazon VPC CNI is automatically + installed into the cluster. For clusters where you want to use an + alternate CNI this option provides a way to specify that the Amazon + VPC CNI should be deleted. You cannot set this to true if you are + using the Amazon VPC CNI addon. + type: boolean + eksClusterName: + description: EKSClusterName allows you to specify the name of the + EKS cluster in AWS. If you don't specify a name then a default name + will be created based on the namespace and name of the managed control + plane. + type: string + encryptionConfig: + description: EncryptionConfig specifies the encryption configuration + for the cluster + properties: + provider: + description: Provider specifies the ARN or alias of the CMK (in + AWS KMS) + type: string + resources: + description: Resources specifies the resources to be encrypted + items: + type: string + type: array + type: object + endpointAccess: + description: Endpoints specifies access to this cluster's control + plane endpoints + properties: + private: + description: Private points VPC-internal control plane access + to the private endpoint + type: boolean + public: + description: Public controls whether control plane endpoints are + publicly accessible + type: boolean + publicCIDRs: + description: PublicCIDRs specifies which blocks can access the + public endpoint + items: + type: string + type: array + type: object + iamAuthenticatorConfig: + description: IAMAuthenticatorConfig allows the specification of any + additional user or role mappings for use when generating the aws-iam-authenticator + configuration. If this is nil the default configuration is still + generated for the cluster. + properties: + mapRoles: + description: RoleMappings is a list of role mappings + items: + description: RoleMapping represents a mapping from a IAM role + to Kubernetes users and groups + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + rolearn: + description: RoleARN is the AWS ARN for the role to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - rolearn + - username + type: object + type: array + mapUsers: + description: UserMappings is a list of user mappings + items: + description: UserMapping represents a mapping from an IAM user + to Kubernetes users and groups + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + userarn: + description: UserARN is the AWS ARN for the user to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - userarn + - username + type: object + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling the managed control plane. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + logging: + description: Logging specifies which EKS Cluster logs should be enabled. + Entries for each of the enabled logs will be sent to CloudWatch + properties: + apiServer: + default: false + description: APIServer indicates if the Kubernetes API Server + log (kube-apiserver) shoulkd be enabled + type: boolean + audit: + default: false + description: Audit indicates if the Kubernetes API audit log should + be enabled + type: boolean + authenticator: + default: false + description: Authenticator indicates if the iam authenticator + log should be enabled + type: boolean + controllerManager: + default: false + description: ControllerManager indicates if the controller manager + (kube-controller-manager) log should be enabled + type: boolean + scheduler: + default: false + description: Scheduler indicates if the Kubernetes scheduler (kube-scheduler) + log should be enabled + type: boolean + required: + - apiServer + - audit + - authenticator + - controllerManager + - scheduler + type: object + networkSpec: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + region: + description: The AWS Region the cluster lives in. + type: string + roleAdditionalPolicies: + description: RoleAdditionalPolicies allows you to attach additional + polices to the control plane role. You must enable the EKSAllowAddRoles + feature flag to incorporate these into the created role. + items: + type: string + type: array + roleName: + description: RoleName specifies the name of IAM role that gives EKS + permission to make API calls. If the role is pre-existing we will + treat it as unmanaged and not delete it on deletion. If the EKSEnableIAM + feature flag is true and no name is supplied then a role is created. + minLength: 2 + type: string + secondaryCidrBlock: + description: SecondaryCidrBlock is the additional CIDR range to use + for pod IPs. Must be within the 100.64.0.0/10 or 198.19.0.0/16 range. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + tokenMethod: + default: iam-authenticator + description: TokenMethod is used to specify the method for obtaining + a client token for communicating with EKS iam-authenticator - obtains + a client token using iam-authentictor aws-cli - obtains a client + token using the AWS CLI Defaults to iam-authenticator + enum: + - iam-authenticator + - aws-cli + type: string + version: + description: Version defines the desired Kubernetes version. If no + version number is supplied then the latest version of Kubernetes + that EKS supports will be used. + minLength: 2 + pattern: ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.?$ + type: string + type: object + status: + description: AWSManagedControlPlaneStatus defines the observed state of + AWSManagedControlPlane + properties: + addons: + description: Addons holds the current status of the EKS addons + items: + description: AddonState represents the state of an addon + properties: + arn: + description: ARN is the AWS ARN of the addon + type: string + createdAt: + description: CreatedAt is the date and time the addon was created + at + format: date-time + type: string + issues: + description: Issues is a list of issue associated with the addon + items: + description: AddonIssue represents an issue with an addon + properties: + code: + description: Code is the issue code + type: string + message: + description: Message is the textual description of the + issue + type: string + resourceIds: + description: ResourceIDs is a list of resource ids for + the issue + items: + type: string + type: array + type: object + type: array + modifiedAt: + description: ModifiedAt is the date and time the addon was last + modified + format: date-time + type: string + name: + description: Name is the name of the addon + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of the IAM role + used for the service account + type: string + status: + description: Status is the status of the addon + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - arn + - name + - version + type: object + type: array + bastion: + description: Bastion holds details of the instance that is used as + a bastion jump box + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + required: + - id + type: object + conditions: + description: Conditions specifies the cpnditions for the managed control + plane + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + externalManagedControlPlane: + default: true + description: ExternalManagedControlPlane indicates to cluster-api + that the control plane is managed by an external service such as + AKS, EKS, GKE, etc. + type: boolean + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains specifies a list fo available availability + zones that can be used + type: object + failureMessage: + description: ErrorMessage indicates that there is a terminal problem + reconciling the state, and will be set to a descriptive error message. + type: string + initialized: + description: Initialized denotes whether or not the control plane + has the uploaded kubernetes config-map. + type: boolean + network: + description: Networks holds details about the AWS networking resources + used by the control plane + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + oidcProvider: + description: OIDCProvider holds the status of the identity provider + for this cluster + properties: + arn: + description: ARN holds the ARN of the provider + type: string + trustPolicy: + description: TrustPolicy contains the boilerplate IAM trust policy + to use for IRSA + type: string + type: object + ready: + default: false + description: Ready denotes that the AWSManagedControlPlane API Server + is ready to receive requests and that the VPC infra is ready. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSManagedControl belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Control plane infrastructure is ready for worker nodes + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the control plane is using + jsonPath: .spec.network.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSManagedControlPlane is the Schema for the awsmanagedcontrolplanes + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedControlPlaneSpec defines the desired state of AWSManagedControlPlane + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + addons: + description: Addons defines the EKS addons to enable with the EKS + cluster. + items: + description: Addon represents a EKS addon + properties: + conflictResolution: + default: none + description: ConflictResolution is used to declare what should + happen if there are parameter conflicts. Defaults to none + enum: + - overwrite + - none + type: string + name: + description: Name is the name of the addon + minLength: 2 + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of an IAM role + to bind to the addons service account + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - name + - version + type: object + type: array + associateOIDCProvider: + default: false + description: AssociateOIDCProvider can be enabled to automatically + create an identity provider for the controller for use with IAM + roles for service accounts + type: boolean + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + disableVPCCNI: + default: false + description: DisableVPCCNI indicates that the Amazon VPC CNI should + be disabled. With EKS clusters the Amazon VPC CNI is automatically + installed into the cluster. For clusters where you want to use an + alternate CNI this option provides a way to specify that the Amazon + VPC CNI should be deleted. You cannot set this to true if you are + using the Amazon VPC CNI addon. + type: boolean + eksClusterName: + description: EKSClusterName allows you to specify the name of the + EKS cluster in AWS. If you don't specify a name then a default name + will be created based on the namespace and name of the managed control + plane. + type: string + encryptionConfig: + description: EncryptionConfig specifies the encryption configuration + for the cluster + properties: + provider: + description: Provider specifies the ARN or alias of the CMK (in + AWS KMS) + type: string + resources: + description: Resources specifies the resources to be encrypted + items: + type: string + type: array + type: object + endpointAccess: + description: Endpoints specifies access to this cluster's control + plane endpoints + properties: + private: + description: Private points VPC-internal control plane access + to the private endpoint + type: boolean + public: + description: Public controls whether control plane endpoints are + publicly accessible + type: boolean + publicCIDRs: + description: PublicCIDRs specifies which blocks can access the + public endpoint + items: + type: string + type: array + type: object + iamAuthenticatorConfig: + description: IAMAuthenticatorConfig allows the specification of any + additional user or role mappings for use when generating the aws-iam-authenticator + configuration. If this is nil the default configuration is still + generated for the cluster. + properties: + mapRoles: + description: RoleMappings is a list of role mappings + items: + description: RoleMapping represents a mapping from a IAM role + to Kubernetes users and groups + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + rolearn: + description: RoleARN is the AWS ARN for the role to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - rolearn + - username + type: object + type: array + mapUsers: + description: UserMappings is a list of user mappings + items: + description: UserMapping represents a mapping from an IAM user + to Kubernetes users and groups + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + userarn: + description: UserARN is the AWS ARN for the user to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - userarn + - username + type: object + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling the managed control plane. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + logging: + description: Logging specifies which EKS Cluster logs should be enabled. + Entries for each of the enabled logs will be sent to CloudWatch + properties: + apiServer: + default: false + description: APIServer indicates if the Kubernetes API Server + log (kube-apiserver) shoulkd be enabled + type: boolean + audit: + default: false + description: Audit indicates if the Kubernetes API audit log should + be enabled + type: boolean + authenticator: + default: false + description: Authenticator indicates if the iam authenticator + log should be enabled + type: boolean + controllerManager: + default: false + description: ControllerManager indicates if the controller manager + (kube-controller-manager) log should be enabled + type: boolean + scheduler: + default: false + description: Scheduler indicates if the Kubernetes scheduler (kube-scheduler) + log should be enabled + type: boolean + required: + - apiServer + - audit + - authenticator + - controllerManager + - scheduler + type: object + network: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + oidcIdentityProviderConfig: + description: IdentityProviderconfig is used to specify the oidc provider + config to be attached with this eks cluster + properties: + clientId: + description: This is also known as audience. The ID for the client + application that makes authentication requests to the OpenID + identity provider. + type: string + groupsClaim: + description: The JWT claim that the provider uses to return your + groups. + type: string + groupsPrefix: + description: 'The prefix that is prepended to group claims to + prevent clashes with existing names (such as system: groups). + For example, the valueoidc: will create group names like oidc:engineering + and oidc:infra.' + type: string + identityProviderConfigName: + description: "The name of the OIDC provider configuration. \n + IdentityProviderConfigName is a required field" + type: string + issuerUrl: + description: The URL of the OpenID identity provider that allows + the API server to discover public signing keys for verifying + tokens. The URL must begin with https:// and should correspond + to the iss claim in the provider's OIDC ID tokens. Per the OIDC + standard, path components are allowed but query parameters are + not. Typically the URL consists of only a hostname, like https://server.example.org + or https://example.com. This URL should point to the level below + .well-known/openid-configuration and must be publicly accessible + over the internet. + type: string + requiredClaims: + additionalProperties: + type: string + description: The key value pairs that describe required claims + in the identity token. If set, each claim is verified to be + present in the token with a matching value. For the maximum + number of claims that you can require, see Amazon EKS service + quotas (https://docs.aws.amazon.com/eks/latest/userguide/service-quotas.html) + in the Amazon EKS User Guide. + type: object + tags: + additionalProperties: + type: string + description: tags to apply to oidc identity provider association + type: object + usernameClaim: + description: The JSON Web Token (JWT) claim to use as the username. + The default is sub, which is expected to be a unique identifier + of the end user. You can choose other claims, such as email + or name, depending on the OpenID identity provider. Claims other + than email are prefixed with the issuer URL to prevent naming + clashes with other plug-ins. + type: string + usernamePrefix: + description: The prefix that is prepended to username claims to + prevent clashes with existing names. If you do not provide this + field, and username is a value other than email, the prefix + defaults to issuerurl#. You can use the value - to disable all + prefixing. + type: string + type: object + region: + description: The AWS Region the cluster lives in. + type: string + roleAdditionalPolicies: + description: RoleAdditionalPolicies allows you to attach additional + polices to the control plane role. You must enable the EKSAllowAddRoles + feature flag to incorporate these into the created role. + items: + type: string + type: array + roleName: + description: RoleName specifies the name of IAM role that gives EKS + permission to make API calls. If the role is pre-existing we will + treat it as unmanaged and not delete it on deletion. If the EKSEnableIAM + feature flag is true and no name is supplied then a role is created. + minLength: 2 + type: string + secondaryCidrBlock: + description: SecondaryCidrBlock is the additional CIDR range to use + for pod IPs. Must be within the 100.64.0.0/10 or 198.19.0.0/16 range. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + tokenMethod: + default: iam-authenticator + description: TokenMethod is used to specify the method for obtaining + a client token for communicating with EKS iam-authenticator - obtains + a client token using iam-authentictor aws-cli - obtains a client + token using the AWS CLI Defaults to iam-authenticator + enum: + - iam-authenticator + - aws-cli + type: string + version: + description: Version defines the desired Kubernetes version. If no + version number is supplied then the latest version of Kubernetes + that EKS supports will be used. + minLength: 2 + pattern: ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.?$ + type: string + type: object + status: + description: AWSManagedControlPlaneStatus defines the observed state of + AWSManagedControlPlane + properties: + addons: + description: Addons holds the current status of the EKS addons + items: + description: AddonState represents the state of an addon + properties: + arn: + description: ARN is the AWS ARN of the addon + type: string + createdAt: + description: CreatedAt is the date and time the addon was created + at + format: date-time + type: string + issues: + description: Issues is a list of issue associated with the addon + items: + description: AddonIssue represents an issue with an addon + properties: + code: + description: Code is the issue code + type: string + message: + description: Message is the textual description of the + issue + type: string + resourceIds: + description: ResourceIDs is a list of resource ids for + the issue + items: + type: string + type: array + type: object + type: array + modifiedAt: + description: ModifiedAt is the date and time the addon was last + modified + format: date-time + type: string + name: + description: Name is the name of the addon + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of the IAM role + used for the service account + type: string + status: + description: Status is the status of the addon + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - arn + - name + - version + type: object + type: array + bastion: + description: Bastion holds details of the instance that is used as + a bastion jump box + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + volumeIDs: + description: IDs of the instance's volumes + items: + type: string + type: array + required: + - id + type: object + conditions: + description: Conditions specifies the cpnditions for the managed control + plane + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + externalManagedControlPlane: + default: true + description: ExternalManagedControlPlane indicates to cluster-api + that the control plane is managed by an external service such as + AKS, EKS, GKE, etc. + type: boolean + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains specifies a list fo available availability + zones that can be used + type: object + failureMessage: + description: ErrorMessage indicates that there is a terminal problem + reconciling the state, and will be set to a descriptive error message. + type: string + identityProviderStatus: + description: IdentityProviderStatus holds the status for associated + identity provider + properties: + arn: + description: ARN holds the ARN of associated identity provider + type: string + status: + description: Status holds current status of associated identity + provider + type: string + type: object + initialized: + description: Initialized denotes whether or not the control plane + has the uploaded kubernetes config-map. + type: boolean + networkStatus: + description: Networks holds details about the AWS networking resources + used by the control plane + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + oidcProvider: + description: OIDCProvider holds the status of the identity provider + for this cluster + properties: + arn: + description: ARN holds the ARN of the provider + type: string + trustPolicy: + description: TrustPolicy contains the boilerplate IAM trust policy + to use for IRSA + type: string + type: object + ready: + default: false + description: Ready denotes that the AWSManagedControlPlane API Server + is ready to receive requests and that the VPC infra is ready. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Cluster to which this AWSManagedControl belongs + jsonPath: .metadata.labels.cluster\.x-k8s\.io/cluster-name + name: Cluster + type: string + - description: Control plane infrastructure is ready for worker nodes + jsonPath: .status.ready + name: Ready + type: string + - description: AWS VPC the control plane is using + jsonPath: .spec.network.vpc.id + name: VPC + type: string + - description: API Endpoint + jsonPath: .spec.controlPlaneEndpoint.host + name: Endpoint + priority: 1 + type: string + - description: Bastion IP address for breakglass access + jsonPath: .status.bastion.publicIp + name: Bastion IP + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSManagedControlPlane is the schema for the Amazon EKS Managed + Control Plane API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedControlPlaneSpec defines the desired state of an + Amazon EKS Cluster. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + addons: + description: Addons defines the EKS addons to enable with the EKS + cluster. + items: + description: Addon represents a EKS addon. + properties: + conflictResolution: + default: none + description: ConflictResolution is used to declare what should + happen if there are parameter conflicts. Defaults to none + enum: + - overwrite + - none + type: string + name: + description: Name is the name of the addon + minLength: 2 + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of an IAM role + to bind to the addons service account + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - name + - version + type: object + type: array + associateOIDCProvider: + default: false + description: AssociateOIDCProvider can be enabled to automatically + create an identity provider for the controller for use with IAM + roles for service accounts + type: boolean + bastion: + description: Bastion contains options to configure the bastion host. + properties: + allowedCIDRBlocks: + description: AllowedCIDRBlocks is a list of CIDR blocks allowed + to access the bastion host. They are set as ingress rules for + the Bastion host's Security Group (defaults to 0.0.0.0/0). + items: + type: string + type: array + ami: + description: AMI will use the specified AMI to boot the bastion. + If not specified, the AMI will default to one picked out in + public space. + type: string + disableIngressRules: + description: DisableIngressRules will ensure there are no Ingress + rules in the bastion host's security group. Requires AllowedCIDRBlocks + to be empty. + type: boolean + enabled: + description: Enabled allows this provider to create a bastion + host instance with a public ip to access the VPC private network. + type: boolean + instanceType: + description: InstanceType will use the specified instance type + for the bastion. If not specified, Cluster API Provider AWS + will use t3.micro for all regions except us-east-1, where t2.micro + will be the default. + type: string + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint used to + communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + disableVPCCNI: + default: false + description: DisableVPCCNI indicates that the Amazon VPC CNI should + be disabled. With EKS clusters the Amazon VPC CNI is automatically + installed into the cluster. For clusters where you want to use an + alternate CNI this option provides a way to specify that the Amazon + VPC CNI should be deleted. You cannot set this to true if you are + using the Amazon VPC CNI addon. + type: boolean + eksClusterName: + description: EKSClusterName allows you to specify the name of the + EKS cluster in AWS. If you don't specify a name then a default name + will be created based on the namespace and name of the managed control + plane. + type: string + encryptionConfig: + description: EncryptionConfig specifies the encryption configuration + for the cluster + properties: + provider: + description: Provider specifies the ARN or alias of the CMK (in + AWS KMS) + type: string + resources: + description: Resources specifies the resources to be encrypted + items: + type: string + type: array + type: object + endpointAccess: + description: Endpoints specifies access to this cluster's control + plane endpoints + properties: + private: + description: Private points VPC-internal control plane access + to the private endpoint + type: boolean + public: + description: Public controls whether control plane endpoints are + publicly accessible + type: boolean + publicCIDRs: + description: PublicCIDRs specifies which blocks can access the + public endpoint + items: + type: string + type: array + type: object + iamAuthenticatorConfig: + description: IAMAuthenticatorConfig allows the specification of any + additional user or role mappings for use when generating the aws-iam-authenticator + configuration. If this is nil the default configuration is still + generated for the cluster. + properties: + mapRoles: + description: RoleMappings is a list of role mappings + items: + description: RoleMapping represents a mapping from a IAM role + to Kubernetes users and groups. + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + rolearn: + description: RoleARN is the AWS ARN for the role to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - rolearn + - username + type: object + type: array + mapUsers: + description: UserMappings is a list of user mappings + items: + description: UserMapping represents a mapping from an IAM user + to Kubernetes users and groups. + properties: + groups: + description: Groups is a list of kubernetes RBAC groups + items: + type: string + type: array + userarn: + description: UserARN is the AWS ARN for the user to map + minLength: 31 + type: string + username: + description: UserName is a kubernetes RBAC user subject + type: string + required: + - groups + - userarn + - username + type: object + type: array + type: object + identityRef: + description: IdentityRef is a reference to a identity to be used when + reconciling the managed control plane. + properties: + kind: + description: Kind of the identity. + enum: + - AWSClusterControllerIdentity + - AWSClusterRoleIdentity + - AWSClusterStaticIdentity + type: string + name: + description: Name of the identity. + minLength: 1 + type: string + required: + - kind + - name + type: object + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating system + used to look up machine images when a machine does not specify an + AMI. When set, this will be used for all cluster machines unless + a machine specifies a different ImageLookupBaseOS. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. Supports substitutions for {{.BaseOS}} + and {{.K8sVersion}} with the base OS and kubernetes version, respectively. + The BaseOS will be the value in ImageLookupBaseOS or ubuntu (the + default), and the kubernetes version as defined by the packages + produced by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to look up + machine images when a machine does not specify an AMI. When set, + this will be used for all cluster machines unless a machine specifies + a different ImageLookupOrg. + type: string + kubeProxy: + description: KubeProxy defines managed attributes of the kube-proxy + daemonset + properties: + disable: + default: false + description: Disable set to true indicates that kube-proxy should + be disabled. With EKS clusters kube-proxy is automatically installed + into the cluster. For clusters where you want to use kube-proxy + functionality that is provided with an alternate CNI, this option + provides a way to specify that the kube-proxy daemonset should + be deleted. You cannot set this to true if you are using the + Amazon kube-proxy addon. + type: boolean + type: object + logging: + description: Logging specifies which EKS Cluster logs should be enabled. + Entries for each of the enabled logs will be sent to CloudWatch + properties: + apiServer: + default: false + description: APIServer indicates if the Kubernetes API Server + log (kube-apiserver) shoulkd be enabled + type: boolean + audit: + default: false + description: Audit indicates if the Kubernetes API audit log should + be enabled + type: boolean + authenticator: + default: false + description: Authenticator indicates if the iam authenticator + log should be enabled + type: boolean + controllerManager: + default: false + description: ControllerManager indicates if the controller manager + (kube-controller-manager) log should be enabled + type: boolean + scheduler: + default: false + description: Scheduler indicates if the Kubernetes scheduler (kube-scheduler) + log should be enabled + type: boolean + required: + - apiServer + - audit + - authenticator + - controllerManager + - scheduler + type: object + network: + description: NetworkSpec encapsulates all things related to AWS network. + properties: + cni: + description: CNI configuration + properties: + cniIngressRules: + description: CNIIngressRules specify rules to apply to control + plane and worker node security groups. The source for the + rule will be set to control plane and worker security group + IDs. + items: + description: CNIIngressRule defines an AWS ingress rule + for CNI requirements. + properties: + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + type: object + securityGroupOverrides: + additionalProperties: + type: string + description: SecurityGroupOverrides is an optional set of security + groups to use for cluster instances This is optional - if not + provided new security groups will be created for the cluster + type: object + subnets: + description: Subnets configuration. + items: + description: SubnetSpec configures an AWS Subnet. + properties: + availabilityZone: + description: AvailabilityZone defines the availability zone + to use for this subnet in the cluster's region. + type: string + cidrBlock: + description: CidrBlock is the CIDR block to be used when + the provider creates a managed VPC. + type: string + id: + description: ID defines a unique identifier to reference + this resource. + type: string + isPublic: + description: IsPublic defines the subnet as a public subnet. + A subnet is public when it is associated with a route + table that has a route to an internet gateway. + type: boolean + natGatewayId: + description: NatGatewayID is the NAT gateway id associated + with the subnet. Ignored unless the subnet is managed + by the provider, in which case this is set on the public + subnet where the NAT gateway resides. It is then used + to determine routes for private subnets in the same AZ + as the public subnet. + type: string + routeTableId: + description: RouteTableID is the routing table id associated + with the subnet. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the + resource. + type: object + type: object + type: array + vpc: + description: VPC configuration. + properties: + availabilityZoneSelection: + default: Ordered + description: 'AvailabilityZoneSelection specifies how AZs + should be selected if there are more AZs in a region than + specified by AvailabilityZoneUsageLimit. There are 2 selection + schemes: Ordered - selects based on alphabetical order Random + - selects AZs randomly in a region Defaults to Ordered' + enum: + - Ordered + - Random + type: string + availabilityZoneUsageLimit: + default: 3 + description: AvailabilityZoneUsageLimit specifies the maximum + number of availability zones (AZ) that should be used in + a region when automatically creating subnets. If a region + has more than this number of AZs then this number of AZs + will be picked randomly when creating default subnets. Defaults + to 3 + minimum: 1 + type: integer + cidrBlock: + description: CidrBlock is the CIDR block to be used when the + provider creates a managed VPC. Defaults to 10.0.0.0/16. + type: string + id: + description: ID is the vpc-id of the VPC this provider should + use to create resources. + type: string + internetGatewayId: + description: InternetGatewayID is the id of the internet gateway + associated with the VPC. + type: string + tags: + additionalProperties: + type: string + description: Tags is a collection of tags describing the resource. + type: object + type: object + type: object + oidcIdentityProviderConfig: + description: IdentityProviderconfig is used to specify the oidc provider + config to be attached with this eks cluster + properties: + clientId: + description: This is also known as audience. The ID for the client + application that makes authentication requests to the OpenID + identity provider. + type: string + groupsClaim: + description: The JWT claim that the provider uses to return your + groups. + type: string + groupsPrefix: + description: 'The prefix that is prepended to group claims to + prevent clashes with existing names (such as system: groups). + For example, the valueoidc: will create group names like oidc:engineering + and oidc:infra.' + type: string + identityProviderConfigName: + description: "The name of the OIDC provider configuration. \n + IdentityProviderConfigName is a required field" + type: string + issuerUrl: + description: The URL of the OpenID identity provider that allows + the API server to discover public signing keys for verifying + tokens. The URL must begin with https:// and should correspond + to the iss claim in the provider's OIDC ID tokens. Per the OIDC + standard, path components are allowed but query parameters are + not. Typically the URL consists of only a hostname, like https://server.example.org + or https://example.com. This URL should point to the level below + .well-known/openid-configuration and must be publicly accessible + over the internet. + type: string + requiredClaims: + additionalProperties: + type: string + description: The key value pairs that describe required claims + in the identity token. If set, each claim is verified to be + present in the token with a matching value. For the maximum + number of claims that you can require, see Amazon EKS service + quotas (https://docs.aws.amazon.com/eks/latest/userguide/service-quotas.html) + in the Amazon EKS User Guide. + type: object + tags: + additionalProperties: + type: string + description: tags to apply to oidc identity provider association + type: object + usernameClaim: + description: The JSON Web Token (JWT) claim to use as the username. + The default is sub, which is expected to be a unique identifier + of the end user. You can choose other claims, such as email + or name, depending on the OpenID identity provider. Claims other + than email are prefixed with the issuer URL to prevent naming + clashes with other plug-ins. + type: string + usernamePrefix: + description: The prefix that is prepended to username claims to + prevent clashes with existing names. If you do not provide this + field, and username is a value other than email, the prefix + defaults to issuerurl#. You can use the value - to disable all + prefixing. + type: string + type: object + partition: + description: Partition is the AWS security partition being used. Defaults + to "aws" + type: string + region: + description: The AWS Region the cluster lives in. + type: string + roleAdditionalPolicies: + description: RoleAdditionalPolicies allows you to attach additional + polices to the control plane role. You must enable the EKSAllowAddRoles + feature flag to incorporate these into the created role. + items: + type: string + type: array + roleName: + description: RoleName specifies the name of IAM role that gives EKS + permission to make API calls. If the role is pre-existing we will + treat it as unmanaged and not delete it on deletion. If the EKSEnableIAM + feature flag is true and no name is supplied then a role is created. + minLength: 2 + type: string + secondaryCidrBlock: + description: SecondaryCidrBlock is the additional CIDR range to use + for pod IPs. Must be within the 100.64.0.0/10 or 198.19.0.0/16 range. + type: string + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to the + bastion host. Valid values are empty string (do not use SSH keys), + a valid SSH key name, or omitted (use the default SSH key name) + type: string + tokenMethod: + default: iam-authenticator + description: TokenMethod is used to specify the method for obtaining + a client token for communicating with EKS iam-authenticator - obtains + a client token using iam-authentictor aws-cli - obtains a client + token using the AWS CLI Defaults to iam-authenticator + enum: + - iam-authenticator + - aws-cli + type: string + version: + description: Version defines the desired Kubernetes version. If no + version number is supplied then the latest version of Kubernetes + that EKS supports will be used. + minLength: 2 + pattern: ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.?$ + type: string + type: object + status: + description: AWSManagedControlPlaneStatus defines the observed state of + an Amazon EKS Cluster. + properties: + addons: + description: Addons holds the current status of the EKS addons + items: + description: AddonState represents the state of an addon. + properties: + arn: + description: ARN is the AWS ARN of the addon + type: string + createdAt: + description: CreatedAt is the date and time the addon was created + at + format: date-time + type: string + issues: + description: Issues is a list of issue associated with the addon + items: + description: AddonIssue represents an issue with an addon. + properties: + code: + description: Code is the issue code + type: string + message: + description: Message is the textual description of the + issue + type: string + resourceIds: + description: ResourceIDs is a list of resource ids for + the issue + items: + type: string + type: array + type: object + type: array + modifiedAt: + description: ModifiedAt is the date and time the addon was last + modified + format: date-time + type: string + name: + description: Name is the name of the addon + type: string + serviceAccountRoleARN: + description: ServiceAccountRoleArn is the ARN of the IAM role + used for the service account + type: string + status: + description: Status is the status of the addon + type: string + version: + description: Version is the version of the addon to use + type: string + required: + - arn + - name + - version + type: object + type: array + bastion: + description: Bastion holds details of the instance that is used as + a bastion jump box + properties: + addresses: + description: Addresses contains the AWS instance associated addresses. + items: + description: MachineAddress contains information for the node's + address. + properties: + address: + description: The machine address. + type: string + type: + description: Machine address type, one of Hostname, ExternalIP + or InternalIP. + type: string + required: + - address + - type + type: object + type: array + availabilityZone: + description: Availability zone of instance + type: string + ebsOptimized: + description: Indicates whether the instance is optimized for Amazon + EBS I/O. + type: boolean + enaSupport: + description: Specifies whether enhanced networking with ENA is + enabled. + type: boolean + iamProfile: + description: The name of the IAM instance profile associated with + the instance, if applicable. + type: string + id: + type: string + imageId: + description: The ID of the AMI used to launch the instance. + type: string + instanceState: + description: The current state of the instance. + type: string + networkInterfaces: + description: Specifies ENIs attached to instance + items: + type: string + type: array + nonRootVolumes: + description: Configuration options for the non root storage volumes. + items: + description: Volume encapsulates the configuration options for + the storage device. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by + the controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage + device. Must be greater than the image snapshot size or + 8 (whichever is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported + for the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + type: array + privateIp: + description: The private IPv4 address assigned to the instance. + type: string + publicIp: + description: The public IPv4 address assigned to the instance, + if applicable. + type: string + rootVolume: + description: Configuration options for the root storage volume. + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + securityGroupIds: + description: SecurityGroupIDs are one or more security group IDs + this instance belongs to. + items: + type: string + type: array + spotMarketOptions: + description: SpotMarketOptions option for configuring instances + to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: The name of the SSH key pair. + type: string + subnetId: + description: The ID of the subnet of the instance. + type: string + tags: + additionalProperties: + type: string + description: The tags associated with the instance. + type: object + tenancy: + description: Tenancy indicates if instance should run on shared + or single-tenant hardware. + type: string + type: + description: The instance type. + type: string + userData: + description: UserData is the raw data script passed to the instance + which is run upon bootstrap. This field must not be base64 encoded + and should only be used when running a new instance. + type: string + volumeIDs: + description: IDs of the instance's volumes + items: + type: string + type: array + required: + - id + type: object + conditions: + description: Conditions specifies the cpnditions for the managed control + plane + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + externalManagedControlPlane: + default: true + description: ExternalManagedControlPlane indicates to cluster-api + that the control plane is managed by an external service such as + AKS, EKS, GKE, etc. + type: boolean + failureDomains: + additionalProperties: + description: FailureDomainSpec is the Schema for Cluster API failure + domains. It allows controllers to understand how many failure + domains a cluster can optionally span across. + properties: + attributes: + additionalProperties: + type: string + description: Attributes is a free form map of attributes an + infrastructure provider might use or require. + type: object + controlPlane: + description: ControlPlane determines if this failure domain + is suitable for use by control plane machines. + type: boolean + type: object + description: FailureDomains specifies a list fo available availability + zones that can be used + type: object + failureMessage: + description: ErrorMessage indicates that there is a terminal problem + reconciling the state, and will be set to a descriptive error message. + type: string + identityProviderStatus: + description: IdentityProviderStatus holds the status for associated + identity provider + properties: + arn: + description: ARN holds the ARN of associated identity provider + type: string + status: + description: Status holds current status of associated identity + provider + type: string + type: object + initialized: + description: Initialized denotes whether or not the control plane + has the uploaded kubernetes config-map. + type: boolean + networkStatus: + description: Networks holds details about the AWS networking resources + used by the control plane + properties: + apiServerElb: + description: APIServerELB is the Kubernetes api server classic + load balancer. + properties: + attributes: + description: Attributes defines extra attributes associated + with the load balancer. + properties: + crossZoneLoadBalancing: + description: CrossZoneLoadBalancing enables the classic + load balancer load balancing. + type: boolean + idleTimeout: + description: IdleTimeout is time that the connection is + allowed to be idle (no data has been sent over the connection) + before it is closed by the load balancer. + format: int64 + type: integer + type: object + availabilityZones: + description: AvailabilityZones is an array of availability + zones in the VPC attached to the load balancer. + items: + type: string + type: array + dnsName: + description: DNSName is the dns name of the load balancer. + type: string + healthChecks: + description: HealthCheck is the classic elb health check associated + with the load balancer. + properties: + healthyThreshold: + format: int64 + type: integer + interval: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + target: + type: string + timeout: + description: A Duration represents the elapsed time between + two instants as an int64 nanosecond count. The representation + limits the largest representable duration to approximately + 290 years. + format: int64 + type: integer + unhealthyThreshold: + format: int64 + type: integer + required: + - healthyThreshold + - interval + - target + - timeout + - unhealthyThreshold + type: object + listeners: + description: Listeners is an array of classic elb listeners + associated with the load balancer. There must be at least + one. + items: + description: ClassicELBListener defines an AWS classic load + balancer listener. + properties: + instancePort: + format: int64 + type: integer + instanceProtocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + port: + format: int64 + type: integer + protocol: + description: ClassicELBProtocol defines listener protocols + for a classic load balancer. + type: string + required: + - instancePort + - instanceProtocol + - port + - protocol + type: object + type: array + name: + description: The name of the load balancer. It must be unique + within the set of load balancers defined in the region. + It also serves as identifier. + type: string + scheme: + description: Scheme is the load balancer scheme, either internet-facing + or private. + type: string + securityGroupIds: + description: SecurityGroupIDs is an array of security groups + assigned to the load balancer. + items: + type: string + type: array + subnetIds: + description: SubnetIDs is an array of subnets in the VPC attached + to the load balancer. + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the load + balancer. + type: object + type: object + securityGroups: + additionalProperties: + description: SecurityGroup defines an AWS security group. + properties: + id: + description: ID is a unique identifier. + type: string + ingressRule: + description: IngressRules is the inbound rules associated + with the security group. + items: + description: IngressRule defines an AWS ingress rule for + security groups. + properties: + cidrBlocks: + description: List of CIDR blocks to allow access from. + Cannot be specified with SourceSecurityGroupID. + items: + type: string + type: array + description: + type: string + fromPort: + format: int64 + type: integer + protocol: + description: SecurityGroupProtocol defines the protocol + type for a security group rule. + type: string + sourceSecurityGroupIds: + description: The security group id to allow access + from. Cannot be specified with CidrBlocks. + items: + type: string + type: array + toPort: + format: int64 + type: integer + required: + - description + - fromPort + - protocol + - toPort + type: object + type: array + name: + description: Name is the security group name. + type: string + tags: + additionalProperties: + type: string + description: Tags is a map of tags associated with the security + group. + type: object + required: + - id + - name + type: object + description: SecurityGroups is a map from the role/kind of the + security group to its unique name, if any. + type: object + type: object + oidcProvider: + description: OIDCProvider holds the status of the identity provider + for this cluster + properties: + arn: + description: ARN holds the ARN of the provider + type: string + trustPolicy: + description: TrustPolicy contains the boilerplate IAM trust policy + to use for IRSA + type: string + type: object + ready: + default: false + description: Ready denotes that the AWSManagedControlPlane API Server + is ready to receive requests and that the VPC infra is ready. + type: boolean + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: awsmanagedmachinepools.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: AWSManagedMachinePool + listKind: AWSManagedMachinePoolList + plural: awsmanagedmachinepools + shortNames: + - awsmmp + singular: awsmanagedmachinepool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: MachinePool ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Number of replicas + jsonPath: .status.replicas + name: Replicas + type: integer + name: v1alpha3 + schema: + openAPIV3Schema: + description: AWSManagedMachinePool is the Schema for the awsmanagedmachinepools + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedMachinePoolSpec defines the desired state of AWSManagedMachinePool + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + amiType: + default: AL2_x86_64 + description: AMIType defines the AMI type + enum: + - AL2_x86_64 + - AL2_x86_64_GPU + - AL2_ARM_64 + type: string + amiVersion: + description: AMIVersion defines the desired AMI release version. If + no version number is supplied then the latest version for the Kubernetes + version will be used + minLength: 2 + type: string + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + diskSize: + description: DiskSize specifies the root disk size + format: int32 + type: integer + eksNodegroupName: + description: EKSNodegroupName specifies the name of the nodegroup + in AWS corresponding to this MachinePool. If you don't specify a + name then a default name will be created based on the namespace + and name of the managed machine pool. + type: string + instanceType: + description: InstanceType specifies the AWS instance type + type: string + labels: + additionalProperties: + type: string + description: Labels specifies labels for the Kubernetes node objects + type: object + providerIDList: + description: ProviderIDList are the provider IDs of instances in the + autoscaling group corresponding to the nodegroup represented by + this machine pool + items: + type: string + type: array + remoteAccess: + description: RemoteAccess specifies how machines can be accessed remotely + properties: + public: + description: Public specifies whether to open port 22 to the public + internet + type: boolean + sourceSecurityGroups: + description: SourceSecurityGroups specifies which security groups + are allowed access + items: + type: string + type: array + sshKeyName: + description: SSHKeyName specifies which EC2 SSH key can be used + to access machines. If left empty, the key from the control + plane is used. + type: string + type: object + roleName: + description: RoleName specifies the name of IAM role for the node + group. If the role is pre-existing we will treat it as unmanaged + and not delete it on deletion. If the EKSEnableIAM feature flag + is true and no name is supplied then a role is created. + type: string + scaling: + description: Scaling specifies scaling for the ASG behind this pool + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup + items: + type: string + type: array + type: object + status: + description: AWSManagedMachinePoolStatus defines the observed state of + AWSManagedMachinePool + properties: + conditions: + description: Conditions defines current service state of the managed + machine pool + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the AWSManagedMachinePool nodegroup + has joined the cluster + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: MachinePool ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Number of replicas + jsonPath: .status.replicas + name: Replicas + type: integer + name: v1alpha4 + schema: + openAPIV3Schema: + description: AWSManagedMachinePool is the Schema for the awsmanagedmachinepools + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedMachinePoolSpec defines the desired state of AWSManagedMachinePool + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + amiType: + default: AL2_x86_64 + description: AMIType defines the AMI type + enum: + - AL2_x86_64 + - AL2_x86_64_GPU + - AL2_ARM_64 + type: string + amiVersion: + description: AMIVersion defines the desired AMI release version. If + no version number is supplied then the latest version for the Kubernetes + version will be used + minLength: 2 + type: string + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + capacityType: + default: onDemand + description: CapacityType specifies the capacity type for the ASG + behind this pool + enum: + - onDemand + - spot + type: string + diskSize: + description: DiskSize specifies the root disk size + format: int32 + type: integer + eksNodegroupName: + description: EKSNodegroupName specifies the name of the nodegroup + in AWS corresponding to this MachinePool. If you don't specify a + name then a default name will be created based on the namespace + and name of the managed machine pool. + type: string + instanceType: + description: InstanceType specifies the AWS instance type + type: string + labels: + additionalProperties: + type: string + description: Labels specifies labels for the Kubernetes node objects + type: object + providerIDList: + description: ProviderIDList are the provider IDs of instances in the + autoscaling group corresponding to the nodegroup represented by + this machine pool + items: + type: string + type: array + remoteAccess: + description: RemoteAccess specifies how machines can be accessed remotely + properties: + public: + description: Public specifies whether to open port 22 to the public + internet + type: boolean + sourceSecurityGroups: + description: SourceSecurityGroups specifies which security groups + are allowed access + items: + type: string + type: array + sshKeyName: + description: SSHKeyName specifies which EC2 SSH key can be used + to access machines. If left empty, the key from the control + plane is used. + type: string + type: object + roleName: + description: RoleName specifies the name of IAM role for the node + group. If the role is pre-existing we will treat it as unmanaged + and not delete it on deletion. If the EKSEnableIAM feature flag + is true and no name is supplied then a role is created. + type: string + scaling: + description: Scaling specifies scaling for the ASG behind this pool + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup + items: + type: string + type: array + taints: + description: Taints specifies the taints to apply to the nodes of + the machine pool + items: + description: Taint defines the specs for a Kubernetes taint. + properties: + effect: + description: Effect specifies the effect for the taint + enum: + - no-schedule + - no-execute + - prefer-no-schedule + type: string + key: + description: Key is the key of the taint + type: string + value: + description: Value is the value of the taint + type: string + required: + - effect + - key + - value + type: object + type: array + type: object + status: + description: AWSManagedMachinePoolStatus defines the observed state of + AWSManagedMachinePool + properties: + conditions: + description: Conditions defines current service state of the managed + machine pool + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the AWSManagedMachinePool nodegroup + has joined the cluster + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + required: + - ready + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: MachinePool ready status + jsonPath: .status.ready + name: Ready + type: string + - description: Number of replicas + jsonPath: .status.replicas + name: Replicas + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: AWSManagedMachinePool is the Schema for the awsmanagedmachinepools + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AWSManagedMachinePoolSpec defines the desired state of AWSManagedMachinePool. + properties: + additionalTags: + additionalProperties: + type: string + description: AdditionalTags is an optional set of tags to add to AWS + resources managed by the AWS provider, in addition to the ones added + by default. + type: object + amiType: + default: AL2_x86_64 + description: AMIType defines the AMI type + enum: + - AL2_x86_64 + - AL2_x86_64_GPU + - AL2_ARM_64 + type: string + amiVersion: + description: AMIVersion defines the desired AMI release version. If + no version number is supplied then the latest version for the Kubernetes + version will be used + minLength: 2 + type: string + availabilityZones: + description: AvailabilityZones is an array of availability zones instances + can run in + items: + type: string + type: array + awsLaunchTemplate: + description: AWSLaunchTemplate specifies the launch template to use + to create the managed node group. If AWSLaunchTemplate is specified, + certain node group configuraions outside of launch template are + prohibited (https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). + properties: + additionalSecurityGroups: + description: AdditionalSecurityGroups is an array of references + to security groups that should be applied to the instances. + These security groups would be set in addition to any security + groups defined at the cluster level or in the actuator. + items: + description: AWSResourceReference is a reference to a specific + AWS resource by ID or filters. Only one of ID or Filters may + be specified. Specifying more than one will result in a validation + error. + properties: + filters: + description: 'Filters is a set of key/value pairs used to + identify a resource They are applied according to the + rules defined by the AWS API: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html' + items: + description: Filter is a filter used to identify an AWS + resource. + properties: + name: + description: Name of the filter. Filter names are + case-sensitive. + type: string + values: + description: Values includes one or more filter values. + Filter values are case-sensitive. + items: + type: string + type: array + required: + - name + - values + type: object + type: array + id: + description: ID of resource + type: string + type: object + type: array + ami: + description: AMI is the reference to the AMI from which to create + the machine instance. + properties: + eksLookupType: + description: EKSOptimizedLookupType If specified, will look + up an EKS Optimized image in SSM Parameter store + enum: + - AmazonLinux + - AmazonLinuxGPU + type: string + id: + description: ID of resource + type: string + type: object + iamInstanceProfile: + description: The name or the Amazon Resource Name (ARN) of the + instance profile associated with the IAM role for the instance. + The instance profile contains the IAM role. + type: string + imageLookupBaseOS: + description: ImageLookupBaseOS is the name of the base operating + system to use for image lookup the AMI is not set. + type: string + imageLookupFormat: + description: 'ImageLookupFormat is the AMI naming format to look + up the image for this machine It will be ignored if an explicit + AMI is set. Supports substitutions for {{.BaseOS}} and {{.K8sVersion}} + with the base OS and kubernetes version, respectively. The BaseOS + will be the value in ImageLookupBaseOS or ubuntu (the default), + and the kubernetes version as defined by the packages produced + by kubernetes/release without v as a prefix: 1.13.0, 1.12.5-mybuild.1, + or 1.17.3. For example, the default image format of capa-ami-{{.BaseOS}}-?{{.K8sVersion}}-* + will end up searching for AMIs that match the pattern capa-ami-ubuntu-?1.18.0-* + for a Machine that is targeting kubernetes v1.18.0 and the ubuntu + base OS. See also: https://golang.org/pkg/text/template/' + type: string + imageLookupOrg: + description: ImageLookupOrg is the AWS Organization ID to use + for image lookup if AMI is not set. + type: string + instanceType: + description: 'InstanceType is the type of instance to create. + Example: m4.xlarge' + type: string + name: + description: The name of the launch template. + type: string + rootVolume: + description: RootVolume encapsulates the configuration options + for the root volume + properties: + deviceName: + description: Device name + type: string + encrypted: + description: Encrypted is whether the volume should be encrypted + or not. + type: boolean + encryptionKey: + description: EncryptionKey is the KMS key to use to encrypt + the volume. Can be either a KMS key ID or ARN. If Encrypted + is set and this is omitted, the default AWS key will be + used. The key must already exist and be accessible by the + controller. + type: string + iops: + description: IOPS is the number of IOPS requested for the + disk. Not applicable to all types. + format: int64 + type: integer + size: + description: Size specifies size (in Gi) of the storage device. + Must be greater than the image snapshot size or 8 (whichever + is greater). + format: int64 + minimum: 8 + type: integer + throughput: + description: Throughput to provision in MiB/s supported for + the volume type. Not applicable to all types. + format: int64 + type: integer + type: + description: Type is the type of the volume (e.g. gp2, io1, + etc...). + type: string + required: + - size + type: object + spotMarketOptions: + description: SpotMarketOptions are options for configuring AWSMachinePool + instances to be run using AWS Spot instances. + properties: + maxPrice: + description: MaxPrice defines the maximum price the user is + willing to pay for Spot VM instances + type: string + type: object + sshKeyName: + description: SSHKeyName is the name of the ssh key to attach to + the instance. Valid values are empty string (do not use SSH + keys), a valid SSH key name, or omitted (use the default SSH + key name) + type: string + versionNumber: + description: 'VersionNumber is the version of the launch template + that is applied. Typically a new version is created when at + least one of the following happens: 1) A new launch template + spec is applied. 2) One or more parameters in an existing template + is changed. 3) A new AMI is discovered.' + format: int64 + type: integer + type: object + capacityType: + default: onDemand + description: CapacityType specifies the capacity type for the ASG + behind this pool + enum: + - onDemand + - spot + type: string + diskSize: + description: DiskSize specifies the root disk size + format: int32 + type: integer + eksNodegroupName: + description: EKSNodegroupName specifies the name of the nodegroup + in AWS corresponding to this MachinePool. If you don't specify a + name then a default name will be created based on the namespace + and name of the managed machine pool. + type: string + instanceType: + description: InstanceType specifies the AWS instance type + type: string + labels: + additionalProperties: + type: string + description: Labels specifies labels for the Kubernetes node objects + type: object + providerIDList: + description: ProviderIDList are the provider IDs of instances in the + autoscaling group corresponding to the nodegroup represented by + this machine pool + items: + type: string + type: array + remoteAccess: + description: RemoteAccess specifies how machines can be accessed remotely + properties: + public: + description: Public specifies whether to open port 22 to the public + internet + type: boolean + sourceSecurityGroups: + description: SourceSecurityGroups specifies which security groups + are allowed access + items: + type: string + type: array + sshKeyName: + description: SSHKeyName specifies which EC2 SSH key can be used + to access machines. If left empty, the key from the control + plane is used. + type: string + type: object + roleAdditionalPolicies: + description: RoleAdditionalPolicies allows you to attach additional + polices to the node group role. You must enable the EKSAllowAddRoles + feature flag to incorporate these into the created role. + items: + type: string + type: array + roleName: + description: RoleName specifies the name of IAM role for the node + group. If the role is pre-existing we will treat it as unmanaged + and not delete it on deletion. If the EKSEnableIAM feature flag + is true and no name is supplied then a role is created. + type: string + scaling: + description: Scaling specifies scaling for the ASG behind this pool + properties: + maxSize: + format: int32 + type: integer + minSize: + format: int32 + type: integer + type: object + subnetIDs: + description: SubnetIDs specifies which subnets are used for the auto + scaling group of this nodegroup + items: + type: string + type: array + taints: + description: Taints specifies the taints to apply to the nodes of + the machine pool + items: + description: Taint defines the specs for a Kubernetes taint. + properties: + effect: + description: Effect specifies the effect for the taint + enum: + - no-schedule + - no-execute + - prefer-no-schedule + type: string + key: + description: Key is the key of the taint + type: string + value: + description: Value is the value of the taint + type: string + required: + - effect + - key + - value + type: object + type: array + updateConfig: + description: UpdateConfig holds the optional config to control the + behaviour of the update to the nodegroup. + properties: + maxUnavailable: + description: MaxUnavailable is the maximum number of nodes unavailable + at once during a version update. Nodes will be updated in parallel. + The maximum number is 100. + maximum: 100 + minimum: 1 + type: integer + maxUnavailablePrecentage: + description: MaxUnavailablePercentage is the maximum percentage + of nodes unavailable during a version update. This percentage + of nodes will be updated in parallel, up to 100 nodes at once. + maximum: 100 + minimum: 1 + type: integer + type: object + type: object + status: + description: AWSManagedMachinePoolStatus defines the observed state of + AWSManagedMachinePool. + properties: + conditions: + description: Conditions defines current service state of the managed + machine pool + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + failureMessage: + description: "FailureMessage will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a more verbose string suitable for logging and human consumption. + \n This field should not be set for transitive errors that a controller + faces that are expected to be fixed automatically over time (like + service outages), but instead indicate that something is fundamentally + wrong with the MachinePool's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + failureReason: + description: "FailureReason will be set in the event that there is + a terminal problem reconciling the MachinePool and will contain + a succinct value suitable for machine interpretation. \n This field + should not be set for transitive errors that a controller faces + that are expected to be fixed automatically over time (like service + outages), but instead indicate that something is fundamentally wrong + with the Machine's spec or the configuration of the controller, + and that manual intervention is required. Examples of terminal errors + would be invalid combinations of settings in the spec, values that + are unsupported by the controller, or the responsible controller + itself being critically misconfigured. \n Any transient errors that + occur during the reconciliation of MachinePools can be added as + events to the MachinePool object and/or logged in the controller's + output." + type: string + ready: + default: false + description: Ready denotes that the AWSManagedMachinePool nodegroup + has joined the cluster + type: boolean + replicas: + description: Replicas is the most recently observed number of replicas. + format: int32 + type: integer + required: + - ready + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: eksconfigs.bootstrap.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: bootstrap.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: EKSConfig + listKind: EKSConfigList + plural: eksconfigs + shortNames: + - eksc + singular: eksconfig + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Bootstrap configuration is ready + jsonPath: .status.ready + name: Ready + type: string + - description: Name of Secret containing bootstrap data + jsonPath: .status.dataSecretName + name: DataSecretName + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + description: EKSConfig is the Schema for the eksconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigSpec defines the desired state of EKSConfig + properties: + kubeletExtraArgs: + additionalProperties: + type: string + description: Passes the kubelet args into the EKS bootstrap script + type: object + type: object + status: + description: EKSConfigStatus defines the observed state of EKSConfig + properties: + conditions: + description: Conditions defines current service state of the EKSConfig. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + dataSecretName: + description: DataSecretName is the name of the secret that stores + the bootstrap data script. + type: string + failureMessage: + description: FailureMessage will be set on non-retryable errors + type: string + failureReason: + description: FailureReason will be set on non-retryable errors + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + ready: + description: Ready indicates the BootstrapData secret is ready to + be consumed + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Bootstrap configuration is ready + jsonPath: .status.ready + name: Ready + type: string + - description: Name of Secret containing bootstrap data + jsonPath: .status.dataSecretName + name: DataSecretName + type: string + name: v1alpha4 + schema: + openAPIV3Schema: + description: EKSConfig is the Schema for the eksconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigSpec defines the desired state of EKSConfig + properties: + kubeletExtraArgs: + additionalProperties: + type: string + description: Passes the kubelet args into the EKS bootstrap script + type: object + type: object + status: + description: EKSConfigStatus defines the observed state of EKSConfig + properties: + conditions: + description: Conditions defines current service state of the EKSConfig. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - status + - type + type: object + type: array + dataSecretName: + description: DataSecretName is the name of the secret that stores + the bootstrap data script. + type: string + failureMessage: + description: FailureMessage will be set on non-retryable errors + type: string + failureReason: + description: FailureReason will be set on non-retryable errors + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + ready: + description: Ready indicates the BootstrapData secret is ready to + be consumed + type: boolean + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Bootstrap configuration is ready + jsonPath: .status.ready + name: Ready + type: string + - description: Name of Secret containing bootstrap data + jsonPath: .status.dataSecretName + name: DataSecretName + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: EKSConfig is the schema for the Amazon EKS Machine Bootstrap + Configuration API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigSpec defines the desired state of Amazon EKS Bootstrap + Configuration. + properties: + apiRetryAttempts: + description: APIRetryAttempts is the number of retry attempts for + AWS API call. + type: integer + containerRuntime: + description: ContainerRuntime specify the container runtime to use + when bootstrapping EKS. + type: string + dnsClusterIP: + description: DNSClusterIP overrides the IP address to use for DNS + queries within the cluster. + type: string + dockerConfigJson: + description: DockerConfigJson is used for the contents of the /etc/docker/daemon.json + file. Useful if you want a custom config differing from the default + one in the AMI. This is expected to be a json string. + type: string + kubeletExtraArgs: + additionalProperties: + type: string + description: KubeletExtraArgs passes the specified kubelet args into + the Amazon EKS machine bootstrap script + type: object + pauseContainer: + description: PauseContainer allows customization of the pause container + to use. + properties: + accountNumber: + description: AccountNumber is the AWS account number to pull the + pause container from. + type: string + version: + description: Version is the tag of the pause container to use. + type: string + required: + - accountNumber + - version + type: object + useMaxPods: + description: UseMaxPods sets --max-pods for the kubelet when true. + type: boolean + type: object + status: + description: EKSConfigStatus defines the observed state of the Amazon + EKS Bootstrap Configuration. + properties: + conditions: + description: Conditions defines current service state of the EKSConfig. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. This should be when the underlying condition changed. + If that is not known, then using the time when the API field + changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. This field may be empty. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. The specific API may choose whether or not this + field is considered a guaranteed API. This field may not be + empty. + type: string + severity: + description: Severity provides an explicit classification of + Reason code, so the users or machines can immediately understand + the current situation and act accordingly. The Severity field + MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources + like Available, but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to deconflict is + important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + dataSecretName: + description: DataSecretName is the name of the secret that stores + the bootstrap data script. + type: string + failureMessage: + description: FailureMessage will be set on non-retryable errors + type: string + failureReason: + description: FailureReason will be set on non-retryable errors + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed + by the controller. + format: int64 + type: integer + ready: + description: Ready indicates the BootstrapData secret is ready to + be consumed + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + controller-gen.kubebuilder.io/version: v0.7.1-0.20211110210727-ab52f76cc7d1 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + cluster.x-k8s.io/v1alpha3: v1alpha3 + cluster.x-k8s.io/v1alpha4: v1alpha4 + cluster.x-k8s.io/v1beta1: v1beta1 + name: eksconfigtemplates.bootstrap.cluster.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: Cg== + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /convert + conversionReviewVersions: + - v1 + - v1beta1 + group: bootstrap.cluster.x-k8s.io + names: + categories: + - cluster-api + kind: EKSConfigTemplate + listKind: EKSConfigTemplateList + plural: eksconfigtemplates + shortNames: + - eksct + singular: eksconfigtemplate + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + description: EKSConfigTemplate is the Schema for the eksconfigtemplates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigTemplateSpec defines the desired state of EKSConfigTemplate + properties: + template: + description: EKSConfigTemplateResource defines the Template structure + properties: + spec: + description: EKSConfigSpec defines the desired state of EKSConfig + properties: + kubeletExtraArgs: + additionalProperties: + type: string + description: Passes the kubelet args into the EKS bootstrap + script + type: object + type: object + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1alpha4 + schema: + openAPIV3Schema: + description: EKSConfigTemplate is the Schema for the eksconfigtemplates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigTemplateSpec defines the desired state of EKSConfigTemplate + properties: + template: + description: EKSConfigTemplateResource defines the Template structure + properties: + spec: + description: EKSConfigSpec defines the desired state of EKSConfig + properties: + kubeletExtraArgs: + additionalProperties: + type: string + description: Passes the kubelet args into the EKS bootstrap + script + type: object + type: object + type: object + required: + - template + type: object + type: object + served: true + storage: false + - name: v1beta1 + schema: + openAPIV3Schema: + description: EKSConfigTemplate is the Amazon EKS Bootstrap Configuration Template + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EKSConfigTemplateSpec defines the desired state of templated + EKSConfig Amazon EKS Bootstrap Configuration resources. + properties: + template: + description: EKSConfigTemplateResource defines the Template structure. + properties: + spec: + description: EKSConfigSpec defines the desired state of Amazon + EKS Bootstrap Configuration. + properties: + apiRetryAttempts: + description: APIRetryAttempts is the number of retry attempts + for AWS API call. + type: integer + containerRuntime: + description: ContainerRuntime specify the container runtime + to use when bootstrapping EKS. + type: string + dnsClusterIP: + description: DNSClusterIP overrides the IP address to use + for DNS queries within the cluster. + type: string + dockerConfigJson: + description: DockerConfigJson is used for the contents of + the /etc/docker/daemon.json file. Useful if you want a custom + config differing from the default one in the AMI. This is + expected to be a json string. + type: string + kubeletExtraArgs: + additionalProperties: + type: string + description: KubeletExtraArgs passes the specified kubelet + args into the Amazon EKS machine bootstrap script + type: object + pauseContainer: + description: PauseContainer allows customization of the pause + container to use. + properties: + accountNumber: + description: AccountNumber is the AWS account number to + pull the pause container from. + type: string + version: + description: Version is the tag of the pause container + to use. + type: string + required: + - accountNumber + - version + type: object + useMaxPods: + description: UseMaxPods sets --max-pods for the kubelet when + true. + type: boolean + type: object + type: object + required: + - template + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awscluster + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awscluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusters + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsclustercontrolleridentity + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsclustercontrolleridentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclustercontrolleridentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsclusterroleidentity + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsclusterroleidentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusterroleidentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsclusterstaticidentity + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsclusterstaticidentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusterstaticidentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsmachine + failurePolicy: Fail + name: mutation.awsmachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmachines + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsfargateprofile + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsfargateprofile.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsfargateprofiles + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmachinepools + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-awsmanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedmachinepools + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-eksconfig + failurePolicy: Fail + matchPolicy: Equivalent + name: default.eksconfigs.bootstrap.cluster.x-k8s.io + rules: + - apiGroups: + - bootstrap.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - eksconfig + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-bootstrap-cluster-x-k8s-io-v1beta1-eksconfigtemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.eksconfigtemplates.bootstrap.cluster.x-k8s.io + rules: + - apiGroups: + - bootstrap.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - eksconfigtemplate + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /mutate-controlplane-cluster-x-k8s-io-v1beta1-awsmanagedcontrolplane + failurePolicy: Fail + matchPolicy: Equivalent + name: default.awsmanagedcontrolplanes.controlplane.cluster.x-k8s.io + rules: + - apiGroups: + - controlplane.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedcontrolplanes + sideEffects: None +--- +apiVersion: v1 +data: + credentials: ${AWS_B64ENCODED_CREDENTIALS} +kind: Secret +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-manager-bootstrap-credentials + namespace: capi-webhook-system +type: Opaque +--- +apiVersion: v1 +kind: Service +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-webhook-service + namespace: capi-webhook-system +spec: + ports: + - port: 443 + targetPort: webhook-server + selector: + cluster.x-k8s.io/provider: infrastructure-aws +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + name: capa-controller-manager + namespace: capi-webhook-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + template: + metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + control-plane: capa-controller-manager + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: ${K8S_CP_LABEL:=node-role.kubernetes.io/control-plane} + operator: Exists + weight: 10 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + weight: 10 + containers: + - args: + - --leader-elect + - --feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true},BootstrapFormatIgnition=${EXP_BOOTSTRAP_FORMAT_IGNITION:=false},ExternalResourceGC=${EXP_EXTERNAL_RESOURCE_GC:=false} + - --v=${CAPA_LOGLEVEL:=0} + - --metrics-bind-addr=127.0.0.1:8080 + image: gcr.io/k8s-staging-cluster-api-aws/cluster-api-aws-controller:latest + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: healthz + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + - containerPort: 9440 + name: healthz + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + fsGroup: 1000 + terminationGracePeriodSeconds: 10 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: capa-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-serving-cert + namespace: capi-webhook-system +spec: + dnsNames: + - capa-webhook-service.capi-webhook-system.svc + - capa-webhook-service.capi-webhook-system.svc.cluster.local + issuerRef: + kind: Issuer + name: capa-selfsigned-issuer + secretName: capa-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-selfsigned-issuer + namespace: capi-webhook-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capa-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-aws + name: capa-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awscluster + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awscluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusters + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsclustercontrolleridentity + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsclustercontrolleridentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclustercontrolleridentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsclusterroleidentity + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsclusterroleidentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusterroleidentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsclusterstaticidentity + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsclusterstaticidentity.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclusterstaticidentities + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsmachine + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmachines + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsmachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmachinetemplate.infrastructure.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsfargateprofile + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsfargateprofile.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsfargateprofiles + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmachinepools + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-infrastructure-cluster-x-k8s-io-v1beta1-awsmanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmanagedmachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedmachinepools + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-eksconfig + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.eksconfigs.bootstrap.cluster.x-k8s.io + rules: + - apiGroups: + - bootstrap.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - eksconfig + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-bootstrap-cluster-x-k8s-io-v1beta1-eksconfigtemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.eksconfigtemplates.bootstrap.cluster.x-k8s.io + rules: + - apiGroups: + - bootstrap.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - eksconfigtemplate + sideEffects: None +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: capa-webhook-service + namespace: capi-webhook-system + path: /validate-controlplane-cluster-x-k8s-io-v1beta1-awsmanagedcontrolplane + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.awsmanagedcontrolplanes.controlplane.cluster.x-k8s.io + rules: + - apiGroups: + - controlplane.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - awsmanagedcontrolplanes + sideEffects: None diff --git a/spectro/global/kustomization.yaml b/spectro/global/kustomization.yaml new file mode 100644 index 0000000000..8713c3ca9d --- /dev/null +++ b/spectro/global/kustomization.yaml @@ -0,0 +1,61 @@ +namePrefix: capa- +namespace: capi-webhook-system + +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-aws" + +resources: + - ../../config/default/credentials.yaml + +bases: + - ../../config/manager + - ../../config/crd + - ../../config/certmanager + - ../../config/webhook + +patchesStrategicMerge: + - ../../config/default/manager_service_account_patch.yaml + - ../../config/default/manager_pull_policy.yaml + - ../../config/default/manager_webhook_patch.yaml + - ../../config/default/webhookcainjection_patch.yaml + - ../../config/default/manager_image_patch.yaml + +configurations: + - ../../config/default/kustomizeconfig.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service + +patchesJson6902: +- target: + group: apps + kind: Deployment + name: controller-manager + namespace: system + version: v1 + path: patch_service_account.yaml diff --git a/spectro/global/patch_service_account.yaml b/spectro/global/patch_service_account.yaml new file mode 100644 index 0000000000..d9cd4321fc --- /dev/null +++ b/spectro/global/patch_service_account.yaml @@ -0,0 +1,2 @@ +- op: remove + path: "/spec/template/spec/serviceAccountName" diff --git a/spectro/run.sh b/spectro/run.sh new file mode 100755 index 0000000000..8dd7c6d0a3 --- /dev/null +++ b/spectro/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +rm ./generated/* + +kustomize build --load_restrictor none global > ./generated/core-global.yaml +kustomize build --load_restrictor none base > ./generated/core-base.yaml \ No newline at end of file diff --git a/templates/cluster-template-eks-ipv6.yaml b/templates/cluster-template-eks-ipv6.yaml new file mode 100644 index 0000000000..07d3b76946 --- /dev/null +++ b/templates/cluster-template-eks-ipv6.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + controlPlaneRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + vpcCni: + env: + - name: ENABLE_PREFIX_DELEGATION + value: "true" + - name: ENABLE_IPv6 + value: "true" + - name: ENABLE_IPv4 + value: "false" + network: + vpc: + enableIPv6: true + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + addons: + - name: "vpc-cni" + version: "v1.11.0-eksbuild.1" + conflictResolution: "overwrite" + - name: "coredns" + version: "v1.8.7-eksbuild.1" + - name: "kube-proxy" + version: "v1.22.6-eksbuild.1" + +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: EKSConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AWSMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AWSMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${AWS_NODE_MACHINE_TYPE}" + iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io" + sshKeyName: "${AWS_SSH_KEY_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: EKSConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: {} diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index dbbed94df3..dfb754078e 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -27,11 +27,11 @@ images: loadBehavior: tryLoad - name: quay.io/jetstack/cert-manager-controller:v1.7.2 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.1.5 + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.2.0 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.1.5 + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.2.0 loadBehavior: tryLoad - - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.1.5 + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.2.0 loadBehavior: tryLoad providers: @@ -49,8 +49,8 @@ providers: new: "imagePullPolicy: IfNotPresent" - old: --metrics-bind-addr=127.0.0.1:8080 new: --metrics-bind-addr=:8080 - - name: v1.1.5 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.5/core-components.yaml" + - name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/core-components.yaml" type: "url" contract: v1beta1 files: @@ -76,8 +76,8 @@ providers: new: "imagePullPolicy: IfNotPresent" - old: --metrics-bind-addr=127.0.0.1:8080 new: --metrics-bind-addr=:8080 - - name: v1.1.5 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.5/bootstrap-components.yaml" + - name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/bootstrap-components.yaml" type: "url" contract: v1beta1 files: @@ -104,8 +104,8 @@ providers: new: "imagePullPolicy: IfNotPresent" - old: --metrics-bind-addr=127.0.0.1:8080 new: --metrics-bind-addr=:8080 - - name: v1.1.5 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. - value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.5/control-plane-components.yaml" + - name: v1.2.0 # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/control-plane-components.yaml" type: "url" contract: v1beta1 files: @@ -202,8 +202,8 @@ variables: EXP_CLUSTER_RESOURCE_SET: "true" EXP_MACHINE_POOL: "true" CLUSTER_TOPOLOGY: "true" - INIT_WITH_BINARY_V1BETA1: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.5/clusterctl-{OS}-{ARCH}" - INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/clusterctl-{OS}-{ARCH}" + INIT_WITH_BINARY_V1BETA1: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.0/clusterctl-{OS}-{ARCH}" + INIT_WITH_BINARY_V1ALPHA4: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}" # INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize # the management cluster to be upgraded. INIT_WITH_KUBERNETES_VERSION: "v1.24.0" diff --git a/test/e2e/data/e2e_eks_conf.yaml b/test/e2e/data/e2e_eks_conf.yaml index b030724132..413731a0e9 100644 --- a/test/e2e/data/e2e_eks_conf.yaml +++ b/test/e2e/data/e2e_eks_conf.yaml @@ -168,6 +168,8 @@ providers: targetName: "cluster-template-eks-machine-deployment-only.yaml" - sourcePath: "./eks/cluster-template-eks-managed-machinepool-only.yaml" targetName: "cluster-template-eks-managed-machinepool-only.yaml" + - sourcePath: "./eks/cluster-template-eks-managed-machinepool-with-launch-template-only.yaml" + targetName: "cluster-template-eks-managed-machinepool-with-launch-template-only.yaml" - sourcePath: "./eks/cluster-template-eks-managedmachinepool.yaml" targetName: "cluster-template-eks-managedmachinepool.yaml" diff --git a/test/e2e/data/e2e_eks_ipv6_conf.yaml b/test/e2e/data/e2e_eks_ipv6_conf.yaml new file mode 100644 index 0000000000..fa6b515bde --- /dev/null +++ b/test/e2e/data/e2e_eks_ipv6_conf.yaml @@ -0,0 +1,231 @@ +--- +# E2E test scenario using local dev images and manifests built from the source tree for following providers: +# - cluster-api +# - infrastructure aws + +# To run tests, run the following from the root of this repository. +# `AWS_REGION=eu-west-1 make e2e GINKGO_ARGS=-stream E2E_ARGS=-skip-cloudformation-deletion` +# The -stream flag will make Ginkgo print results to the screen in real-time. +# -skip-cloudformation-deletion reduces the time taken to set up AWS CloudFormation prior to cluster start. + +# AWS credentials must be present for running tests +# For more details, run `go run ./cmd/clusterawsadm bootstrap credentials` + +images: + # Use local dev images built source tree; + - name: gcr.io/k8s-staging-cluster-api/capa-manager:e2e + loadBehavior: mustLoad + +## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS + - name: quay.io/jetstack/cert-manager-cainjector:v1.7.2 + loadBehavior: tryLoad + - name: quay.io/jetstack/cert-manager-webhook:v1.7.2 + loadBehavior: tryLoad + - name: quay.io/jetstack/cert-manager-controller:v1.7.2 + loadBehavior: tryLoad + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.1.2 + loadBehavior: tryLoad + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.1.2 + loadBehavior: tryLoad + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.1.2 + loadBehavior: tryLoad + + +providers: + - name: cluster-api + type: CoreProvider + versions: + - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. + contract: v1alpha3 + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/core-components.yaml" + type: "url" + files: + - sourcePath: "./shared/v1alpha3/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. + contract: v1alpha4 + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/core-components.yaml" + type: "url" + files: + - sourcePath: "./shared/v1alpha4/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.2 + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.99 # next; + value: "https://storage.googleapis.com/artifacts.k8s-staging-cluster-api.appspot.com/components/nightly_main_20220216/core-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: kubeadm + type: BootstrapProvider + files: + - sourcePath: "./shared/v1alpha4/metadata.yaml" + versions: + - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/bootstrap-components.yaml" + type: "url" + contract: v1alpha3 + files: + - sourcePath: "./shared/v1alpha3/metadata.yaml" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/bootstrap-components.yaml" + type: "url" + contract: v1alpha4 + files: + - sourcePath: "./shared/v1alpha4/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.2 + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.99 # next; + value: "https://storage.googleapis.com/artifacts.k8s-staging-cluster-api.appspot.com/components/nightly_main_20220216/bootstrap-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: kubeadm + type: ControlPlaneProvider + files: + - sourcePath: "./shared/v1alpha4/metadata.yaml" + versions: + - name: v0.3.23 # latest published release in the v1alpha3 series; this is used for v1alpha3 --> v1beta1 clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/control-plane-components.yaml" + type: "url" + contract: v1alpha3 + files: + - sourcePath: "./shared/v1alpha3/metadata.yaml" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + - name: v0.4.7 # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only. + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.7/control-plane-components.yaml" + type: "url" + contract: v1alpha4 + files: + - sourcePath: "./shared/v1alpha4/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.2 + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: v1.1.99 # next; + value: "https://storage.googleapis.com/artifacts.k8s-staging-cluster-api.appspot.com/components/nightly_main_20220216/control-plane-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - name: aws + type: InfrastructureProvider + versions: + - name: v1.2.99 + # Use manifest from source files + value: ../../../config/default + contract: v1beta1 + files: + - sourcePath: "./shared/v1beta1_provider/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: --metrics-bind-addr=127.0.0.1:8080 + new: --metrics-bind-addr=:8080 + - old: gcr.io/k8s-staging-cluster-api/cluster-api-aws-controller-amd64:dev + new: gcr.io/k8s-staging-cluster-api/capa-manager:e2e + - old: gcr.io/k8s-staging-cluster-api-aws/cluster-api-aws-controller:latest + new: gcr.io/k8s-staging-cluster-api/capa-manager:e2e + files: + - sourcePath: "./eks/cluster-template-eks-ipv6-cluster" + targetName: "cluster-template-eks-ipv6-cluster.yaml" + +variables: + KUBERNETES_VERSION: "v1.22.9" + KUBERNETES_VERSION_MANAGEMENT: "v1.22.9" # Kind bootstrap + EXP_MACHINE_POOL: "true" + EXP_CLUSTER_RESOURCE_SET: "true" + AWS_NODE_MACHINE_TYPE: t3.large + AWS_MACHINE_TYPE_VCPU_USAGE: 2 + AWS_SSH_KEY_NAME: "cluster-api-provider-aws-sigs-k8s-io" + EXP_EKS_IAM: "false" + EXP_EKS_ADD_ROLES: "false" + VPC_ADDON_VERSION: "v1.11.0-eksbuild.1" + COREDNS_ADDON_VERSION: "v1.8.7-eksbuild.1" + KUBE_PROXY_ADDON_VERSION: "v1.22.6-eksbuild.1" + ENABLE_IPV6: "true" + CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "1.22.9" + AUTO_CONTROLLER_IDENTITY_CREATOR: "false" + IP_FAMILY: "IPv6" + +intervals: + default/wait-cluster: ["30m", "10s"] + default/wait-control-plane: ["30m", "10s"] + default/wait-worker-nodes: ["30m", "10s"] + default/wait-controllers: ["5m", "10s"] + default/wait-delete-cluster: ["35m", "30s"] + default/wait-delete-machine: ["10m", "10s"] + default/wait-delete-machine-deployment: ["10m", "10s"] + default/wait-delete-machine-pool: ["20m", "10s"] + default/wait-machine-upgrade: ["20m", "10s"] + default/wait-machine-status: ["20m", "10s"] + default/wait-infra-subnets: ["5m", "30s"] + default/wait-control-plane-upgrade: ["35m", "30s"] + default/wait-addon-status: ["10m", "30s"] + default/wait-create-identity: ["1m", "10s"] diff --git a/test/e2e/data/eks/cluster-template-eks-control-plane-only-withaddon.yaml b/test/e2e/data/eks/cluster-template-eks-control-plane-only-withaddon.yaml index 65e91031b8..f815891055 100644 --- a/test/e2e/data/eks/cluster-template-eks-control-plane-only-withaddon.yaml +++ b/test/e2e/data/eks/cluster-template-eks-control-plane-only-withaddon.yaml @@ -24,6 +24,12 @@ spec: region: "${AWS_REGION}" sshKeyName: "${AWS_SSH_KEY_NAME}" version: "${KUBERNETES_VERSION}" + VpcCni: + env: + - name: FOO + value: BAR + - name: ENABLE_PREFIX_DELEGATION + value: "true" addons: - name: "vpc-cni" version: "${VPC_ADDON_VERSION}" diff --git a/test/e2e/data/eks/cluster-template-eks-ipv6-cluster.yaml b/test/e2e/data/eks/cluster-template-eks-ipv6-cluster.yaml new file mode 100644 index 0000000000..89b7f4fab7 --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-ipv6-cluster.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + controlPlaneRef: + kind: AWSManagedControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +kind: AWSManagedControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + vpcCni: + env: + - name: ENABLE_PREFIX_DELEGATION + value: "true" + - name: ENABLE_IPV6 + value: "true" + - name: ENABLE_IPV4 + value: "false" + network: + vpc: + enableIPv6: "${ENABLE_IPV6}" + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" + version: "${KUBERNETES_VERSION}" + addons: + - name: "vpc-cni" + version: "${VPC_ADDON_VERSION}" + conflictResolution: "overwrite" + - name: "coredns" + version: "${COREDNS_ADDON_VERSION}" + conflictResolution: "overwrite" + - name: "kube-proxy" + version: "${KUBE_PROXY_ADDON_VERSION}" + conflictResolution: "overwrite" + identityRef: + kind: AWSClusterStaticIdentity + name: e2e-account + # secondaryCidrBlock added to test it's creation and deletion functionality within the cluster. + secondaryCidrBlock: 100.64.0.0/16 diff --git a/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-launch-template-only.yaml b/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-launch-template-only.yaml new file mode 100644 index 0000000000..7c157a413a --- /dev/null +++ b/test/e2e/data/eks/cluster-template-eks-managed-machinepool-with-launch-template-only.yaml @@ -0,0 +1,37 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: "${CLUSTER_NAME}-pool-lt-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + version: "EKS_KUBERNETES_VERSION" + clusterName: "${CLUSTER_NAME}" + bootstrap: + dataSecretName: "${CLUSTER_NAME}-pool-lt-0-userdata" + infrastructureRef: + name: "${CLUSTER_NAME}-pool-lt-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AWSManagedMachinePool +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AWSManagedMachinePool +metadata: + name: "${CLUSTER_NAME}-pool-lt-0" +spec: + amiType: CUSTOM + awsLaunchTemplate: + ami: {} + scaling: + minSize: 1 + maxSize: 2 +--- +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-pool-lt-0-userdata" +data: + value: "USER_DATA" +type: Opaque \ No newline at end of file diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/clusterpolicy-crd.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/clusterpolicy-crd.yaml index 1eabc1a5db..8a23622667 100644 --- a/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/clusterpolicy-crd.yaml +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/clusterpolicy-crd.yaml @@ -15,5759 +15,3862 @@ spec: singular: clusterpolicy scope: Cluster versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterPolicy is the Schema for the clusterpolicies API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation + - name: v1 + schema: + openAPIV3Schema: + description: ClusterPolicy is the Schema for the clusterpolicies API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterPolicySpec defines the desired state of ClusterPolicy - properties: - dcgmExporter: - description: DCGMExporter spec - properties: - affinity: - description: 'Optional: Set Node affinity' + type: string + metadata: + type: object + spec: + description: ClusterPolicySpec defines the desired state of ClusterPolicy + properties: + daemonsets: + description: Daemonset defines common configuration for all Daemonsets + properties: + priorityClassName: + type: string + tolerations: + description: 'Optional: Set tolerations' + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + type: object + dcgm: + description: DCGM component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA DCGM Hostengine + as a separate pod is enabled. + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + configMapKeyRef: + description: Selects a key of a ConfigMap. properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - args: - description: 'Optional: List of arguments' - items: - type: string - type: array - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in the - container and any service environment variables. If a - variable cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax can be - escaped with a double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the pod: supports metadata.name, + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of the container: only + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - image: - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: Image pull policy - type: string - imagePullSecrets: - description: Image pull secrets - items: - type: string - type: array - licensingConfig: - description: 'Optional: Licensing configuration for vGPU drivers' - properties: - configMapName: - type: string - type: object - nodeSelector: - additionalProperties: - type: string - description: Node selector to control the selection of nodes (optional) - type: object - podSecurityContext: - description: 'Optional: Pod Security Context' - properties: - fsGroup: - description: "A special supplemental group that applies to - all containers in a pod. Some volume types allow the Kubelet - to change the ownership of that volume to be owned by the - pod: \n 1. The owning GID will be the FSGroup 2. The setgid - bit is set (new files created in the volume will be owned - by FSGroup) 3. The permission bits are OR'd with rw-rw---- - \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will - have no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - SecurityContext. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process - run in each container, in addition to the container's primary - GID. If unspecified, no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options within a container's - SecurityContext will be used. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - repoConfig: - description: 'Optional: Custom repo configuration for driver container' - properties: - configMapName: - type: string - destinationDir: - type: string - type: object - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' + type: array + hostPort: + description: 'HostPort represents host port that needs to be bound + for DCGM engine (Default: 5555)' + format: int32 + type: integer + image: + description: NVIDIA DCGM image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - resources: - description: 'Optional: Define resources requests and limits for + type: array + repository: + description: NVIDIA DCGM image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each pod' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Optional: Security Context' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a - process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in - privileged containers are essentially equivalent to root - on the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use - for the containers. The default is DefaultProcMount which - uses the container runtime defaults for readonly paths and - masked paths. This requires the ProcMountType feature flag - to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - tolerations: - description: 'Optional: Set tolerations' - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, allowed - values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to - the value. Valid operators are Exists and Equal. Defaults - to Equal. Exists is equivalent to wildcard for value, - so that a pod can tolerate all taints of a particular - category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of - time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the taint - forever (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - type: array - version: - pattern: '[a-zA-Z0-9\.-]+' + type: object + version: + description: NVIDIA DCGM image tag + type: string + type: object + dcgmExporter: + description: DCGMExporter spec + properties: + args: + description: 'Optional: List of arguments' + items: type: string - required: - - image - - repository - - version - type: object - devicePlugin: - description: DevicePlugin component spec - properties: - affinity: - description: 'Optional: Set Node affinity' + type: array + config: + description: 'Optional: Custom metrics configuration for NVIDIA + DCGM Exporter' + properties: + name: + description: ConfigMap name with file dcgm-metrics.csv for + metrics to be collected by NVIDIA DCGM Exporter + type: string + type: object + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + configMapKeyRef: + description: Selects a key of a ConfigMap. properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - args: - description: 'Optional: List of arguments' - items: - type: string - type: array - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in the - container and any service environment variables. If a - variable cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax can be - escaped with a double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the pod: supports metadata.name, + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of the container: only + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - image: - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: Image pull policy - type: string - imagePullSecrets: - description: Image pull secrets - items: - type: string - type: array - licensingConfig: - description: 'Optional: Licensing configuration for vGPU drivers' - properties: - configMapName: - type: string - type: object - nodeSelector: - additionalProperties: - type: string - description: Node selector to control the selection of nodes (optional) - type: object - podSecurityContext: - description: 'Optional: Pod Security Context' - properties: - fsGroup: - description: "A special supplemental group that applies to - all containers in a pod. Some volume types allow the Kubelet - to change the ownership of that volume to be owned by the - pod: \n 1. The owning GID will be the FSGroup 2. The setgid - bit is set (new files created in the volume will be owned - by FSGroup) 3. The permission bits are OR'd with rw-rw---- - \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will - have no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - SecurityContext. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process - run in each container, in addition to the container's primary - GID. If unspecified, no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options within a container's - SecurityContext will be used. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - repoConfig: - description: 'Optional: Custom repo configuration for driver container' - properties: - configMapName: - type: string - destinationDir: - type: string - type: object - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' + type: array + image: + description: NVIDIA DCGM Exporter image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - resources: - description: 'Optional: Define resources requests and limits for + type: array + repository: + description: NVIDIA DCGM Exporter image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each pod' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Optional: Security Context' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a - process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in - privileged containers are essentially equivalent to root - on the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use - for the containers. The default is DefaultProcMount which - uses the container runtime defaults for readonly paths and - masked paths. This requires the ProcMountType feature flag - to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - tolerations: - description: 'Optional: Set tolerations' - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, allowed - values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to - the value. Valid operators are Exists and Equal. Defaults - to Equal. Exists is equivalent to wildcard for value, - so that a pod can tolerate all taints of a particular - category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of - time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the taint - forever (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - type: array - version: - pattern: '[a-zA-Z0-9\.-]+' + type: object + version: + description: NVIDIA DCGM Exporter image tag + type: string + type: object + devicePlugin: + description: DevicePlugin component spec + properties: + args: + description: 'Optional: List of arguments' + items: type: string - required: - - image - - repository - - version - type: object - driver: - description: Driver component spec - properties: - affinity: - description: 'Optional: Set Node affinity' + type: array + config: + description: 'Optional: Configuration for the NVIDIA Device Plugin + via the ConfigMap' + properties: + default: + description: Default config name within the ConfigMap for + the NVIDIA Device Plugin config + type: string + name: + description: ConfigMap name for NVIDIA Device Plugin config + including shared config between plugin and GFD + type: string + type: object + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + configMapKeyRef: + description: Selects a key of a ConfigMap. properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - args: - description: 'Optional: List of arguments' - items: - type: string - type: array - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in the - container and any service environment variables. If a - variable cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax can be - escaped with a double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the pod: supports metadata.name, + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of the container: only + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - image: - pattern: '[a-zA-Z0-9\-]+' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: NVIDIA Device Plugin image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - imagePullPolicy: - description: Image pull policy + type: array + repository: + description: NVIDIA Device Plugin image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: NVIDIA Device Plugin image tag + type: string + type: object + driver: + description: Driver component spec + properties: + args: + description: 'Optional: List of arguments' + items: type: string - imagePullSecrets: - description: Image pull secrets - items: + type: array + certConfig: + description: 'Optional: Custom certificates configuration for + NVIDIA Driver container' + properties: + name: type: string - type: array - licensingConfig: - description: 'Optional: Licensing configuration for vGPU drivers' + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA Driver + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - configMapName: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. type: string - type: object - nodeSelector: - additionalProperties: - type: string - description: Node selector to control the selection of nodes (optional) - type: object - podSecurityContext: - description: 'Optional: Pod Security Context' - properties: - fsGroup: - description: "A special supplemental group that applies to - all containers in a pod. Some volume types allow the Kubelet - to change the ownership of that volume to be owned by the - pod: \n 1. The owning GID will be the FSGroup 2. The setgid - bit is set (new files created in the volume will be owned - by FSGroup) 3. The permission bits are OR'd with rw-rw---- - \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will - have no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' type: string - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - SecurityContext. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. + required: + - name + type: object + type: array + image: + description: NVIDIA Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + kernelModuleConfig: + description: 'Optional: Kernel module configuration parameters + for the NVIDIA Driver' + properties: + name: + type: string + type: object + licensingConfig: + description: 'Optional: Licensing configuration for NVIDIA vGPU + licensing' + properties: + configMapName: + type: string + nlsEnabled: + description: NLSEnabled indicates if NVIDIA Licensing System + is used for licensing. + type: boolean + type: object + manager: + description: Manager represents configuration for NVIDIA Driver + Manager initContainer + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object required: - - type + - name type: object - supplementalGroups: - description: A list of groups applied to the first process - run in each container, in addition to the container's primary - GID. If unspecified, no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options within a container's - SecurityContext will be used. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository + path + type: string + version: + description: Version represents NVIDIA Driver Manager image + tag(version) + type: string + type: object + rdma: + description: GPUDirectRDMASpec defines the properties for nvidia-peermem + deployment + properties: + enabled: + description: Enabled indicates if GPUDirect RDMA is enabled + through GPU operator + type: boolean + useHostMofed: + description: UseHostMOFED indicates to use MOFED drivers directly + installed on the host to enable GPUDirect RDMA + type: boolean + type: object + repoConfig: + description: 'Optional: Custom repo configuration for NVIDIA Driver + container' + properties: + configMapName: + type: string + type: object + repository: + description: NVIDIA Driver image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + rollingUpdate: + description: 'Optional: Configuration for rolling update of NVIDIA + Driver DaemonSet pods' + properties: + maxUnavailable: + type: string + type: object + version: + description: NVIDIA Driver image tag + type: string + virtualTopology: + description: 'Optional: Virtual Topology Daemon configuration + for NVIDIA vGPU drivers' + properties: + config: + description: 'Optional: Config name representing virtual topology + daemon configuration file nvidia-topologyd.conf' + type: string + type: object + type: object + gds: + description: GPUDirectStorage defines the spec for GDS components(Experimental) + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if GPUDirect Storage is enabled + through GPU operator + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - repoConfig: - description: 'Optional: Custom repo configuration for driver container' + type: array + image: + description: NVIDIA GPUDirect Storage Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA GPUDirect Storage Driver image repository + type: string + version: + description: NVIDIA GPUDirect Storage Driver image tag + type: string + type: object + gfd: + description: GPUFeatureDiscovery spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - configMapName: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. type: string - destinationDir: + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name type: object - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' + type: array + image: + description: GFD image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - resources: - description: 'Optional: Define resources requests and limits for + type: array + repository: + description: GFD image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: GFD image tag + type: string + type: object + mig: + description: MIG spec + properties: + strategy: + description: 'Optional: MIGStrategy to apply for GFD and NVIDIA + Device Plugin' + enum: + - none + - single + - mixed + type: string + type: object + migManager: + description: MIGManager for configuration to deploy MIG Manager + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: 'Optional: Custom mig-parted configuration for NVIDIA + MIG Manager container' + properties: + name: + description: ConfigMap name + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA MIG Manager + is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + required: + - name + type: object + type: array + gpuClientsConfig: + description: 'Optional: Custom gpu-clients configuration for NVIDIA + MIG Manager container' + properties: + name: + description: ConfigMap name + type: string + type: object + image: + description: NVIDIA MIG Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA MIG Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: NVIDIA MIG Manager image tag + type: string + type: object + nodeStatusExporter: + description: NodeStatusExporter spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of Node Status Exporter + is enabled. + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - securityContext: - description: 'Optional: Security Context' + type: array + image: + description: Node Status Exporter image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Node Status Exporterimage repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: Node Status Exporterimage tag + type: string + type: object + operator: + description: Operator component spec + properties: + defaultRuntime: + default: docker + description: Runtime defines container runtime type + enum: + - docker + - crio + - containerd + type: string + initContainer: + description: InitContainerSpec describes configuration for initContainer + image used with all components + properties: + image: + description: Image represents image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents image repository path + type: string + version: + description: Version represents image tag(version) + type: string + type: object + runtimeClass: + default: nvidia + type: string + use_ocp_driver_toolkit: + description: UseOpenShiftDriverToolkit indicates if DriverToolkit + image should be used on OpenShift to build and install driver + modules + type: boolean + required: + - defaultRuntime + type: object + psp: + description: PSP defines spec for handling PodSecurityPolicies + properties: + enabled: + description: Enabled indicates if PodSecurityPolicies needs to + be enabled for all Pods + type: boolean + type: object + sandboxDevicePlugin: + description: SandboxDevicePlugin component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA Sandbox + Device Plugin through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a - process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the - container runtime. + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object - privileged: - description: Run container in privileged mode. Processes in - privileged containers are essentially equivalent to root - on the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use - for the containers. The default is DefaultProcMount which - uses the container runtime defaults for readonly paths and - masked paths. This requires the ProcMountType feature flag - to be enabled. + required: + - name + type: object + type: array + image: + description: NVIDIA Sandbox Device Plugin image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA Sandbox Device Plugin image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: NVIDIA Sandbox Device Plugin image tag + type: string + type: object + sandboxWorkloads: + description: SandboxWorkloads defines the spec for handling sandbox + workloads (i.e. Virtual Machines) + properties: + defaultWorkload: + default: container + description: DefaultWorkload indicates the default GPU workload + type to configure worker nodes in the cluster for + enum: + - container + - vm-passthrough + - vm-vgpu + type: string + enabled: + description: Enabled indicates if the GPU Operator should manage + additional operands required for sandbox workloads (i.e. VFIO + Manager, vGPU Manager, and additional device plugins) + type: boolean + type: object + toolkit: + description: Toolkit component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA Container + Toolkit through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. + required: + - name + type: object + type: array + image: + description: NVIDIA Container Toolkit image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA Container Toolkit image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: NVIDIA Container Toolkit image tag + type: string + type: object + validator: + description: Validator defines the spec for operator-validator daemonset + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + cuda: + description: CUDA validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object required: - - type + - name type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. + type: array + type: object + driver: + description: Toolkit validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name type: object - type: object - tolerations: - description: 'Optional: Set tolerations' - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, allowed - values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to - the value. Valid operators are Exists and Equal. Defaults - to Equal. Exists is equivalent to wildcard for value, - so that a pod can tolerate all taints of a particular - category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of - time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the taint - forever (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string - type: object - type: array - version: - pattern: '[a-zA-Z0-9\.-]+' - type: string - required: - - image - - repository - - version - type: object - gfd: - description: GPUFeatureDiscovery spec - properties: - affinity: - description: 'Optional: Set Node affinity' + type: array + type: object + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean required: - - nodeSelectorTerms + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: Validator image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + plugin: + description: Plugin validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + type: object + repository: + description: Validator image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + toolkit: + description: Toolkit validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + type: object + version: + description: Validator image tag + type: string + vfioPCI: + description: VfioPCI validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + type: object + vgpuDevices: + description: VGPUDevices validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + type: object + vgpuManager: + description: VGPUManager validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). + type: array + type: object + type: object + vfioManager: + description: VFIOManager for configuration to deploy VFIO-PCI Manager + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + driverManager: + description: DriverManager represents configuration for NVIDIA + Driver Manager + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name type: object - type: object - args: - description: 'Optional: List of arguments' - items: + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository + path + type: string + version: + description: Version represents NVIDIA Driver Manager image + tag(version) type: string - type: array - discoveryIntervalSeconds: - description: 'Optional: Discovery Interval for GPU feature discovery - plugin' - type: integer - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in the - container and any service environment variables. If a - variable cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax can be - escaped with a double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: object + enabled: + description: Enabled indicates if deployment of VFIO Manager is + enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the pod: supports metadata.name, + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of the container: only + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - image: - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: Image pull policy - type: string - imagePullSecrets: - description: Image pull secrets - items: - type: string - type: array - migStrategy: - description: 'Optional: MigStrategy for GPU feature discovery - plugin' - enum: - - none - - single - - mixed - type: string - nodeSelector: - additionalProperties: - type: string - description: Node selector to control the selection of nodes (optional) - type: object - podSecurityContext: - description: 'Optional: Pod Security Context' - properties: - fsGroup: - description: "A special supplemental group that applies to - all containers in a pod. Some volume types allow the Kubelet - to change the ownership of that volume to be owned by the - pod: \n 1. The owning GID will be the FSGroup 2. The setgid - bit is set (new files created in the volume will be owned - by FSGroup) 3. The permission bits are OR'd with rw-rw---- - \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will - have no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' - type: string - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - SecurityContext. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process - run in each container, in addition to the container's primary - GID. If unspecified, no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options within a container's - SecurityContext will be used. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' + type: array + image: + description: VFIO Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - resources: - description: 'Optional: Define resources requests and limits for + type: array + repository: + description: VFIO Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each pod' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Optional: Security Context' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a - process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in - privileged containers are essentially equivalent to root - on the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use - for the containers. The default is DefaultProcMount which - uses the container runtime defaults for readonly paths and - masked paths. This requires the ProcMountType feature flag - to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - tolerations: - description: 'Optional: Set tolerations' - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, allowed - values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to - the value. Valid operators are Exists and Equal. Defaults - to Equal. Exists is equivalent to wildcard for value, - so that a pod can tolerate all taints of a particular - category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of - time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the taint - forever (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - type: array - version: - pattern: '[a-zA-Z0-9\.-]+' - type: string - required: - - image - - repository - - version - type: object - operator: - description: Operator component spec - properties: - defaultRuntime: - description: Runtime defines container runtime type - enum: - - docker - - crio - - containerd + type: object + version: + description: VFIO Manager image tag + type: string + type: object + vgpuDeviceManager: + description: VGPUDeviceManager spec + properties: + args: + description: 'Optional: List of arguments' + items: type: string - validator: - description: ValidatorSpec describes configuration options for - validation pod + type: array + config: + description: NVIDIA vGPU devices configuration for NVIDIA vGPU + Device Manager container + properties: + default: + default: default + description: Default config name within the ConfigMap + type: string + name: + default: vgpu-devices-config + description: ConfigMap name + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA vGPU Device + Manager is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - image: - pattern: '[a-zA-Z0-9\-]+' + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. type: string - imagePullPolicy: - description: Image pull policy + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' type: string - imagePullSecrets: - description: Image pull secrets - items: - type: string - type: array - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' - type: string - version: - pattern: '[a-zA-Z0-9\.-]+' - type: string - type: object - required: - - defaultRuntime - type: object - toolkit: - description: Toolkit component spec - properties: - affinity: - description: 'Optional: Set Node affinity' - properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node matches the corresponding matchExpressions; - the node(s) with the highest sum are the most preferred. - items: - description: An empty preferred scheduling term matches - all objects with implicit weight 0 (i.e. it's a no-op). - A null preferred scheduling term matches no objects - (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from - its node. + configMapKeyRef: + description: Selects a key of a ConfigMap. properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. - items: - description: A null or empty node selector term - matches no objects. The requirements of them are - ANDed. The TopologySelectorTerm type implements - a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: A node selector requirement is - a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: Represents a key's relationship - to a set of values. Valid operators - are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. - type: string - values: - description: An array of string values. - If the operator is In or NotIn, the - values array must be non-empty. If the - operator is Exists or DoesNotExist, - the values array must be empty. If the - operator is Gt or Lt, the values array - must have a single element, which will - be interpreted as an integer. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the affinity expressions specified - by this field, but it may choose a node that violates - one or more of the expressions. The node that is most - preferred is the one with the greatest sum of weights, - i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by - this field are not met at scheduling time, the pod will - not be scheduled onto the node. If the affinity requirements - specified by this field cease to be met at some point - during pod execution (e.g. due to a pod label update), - the system may or may not try to eventually evict the - pod from its node. When there are multiple elements, - the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods - to nodes that satisfy the anti-affinity expressions - specified by this field, but it may choose a node that - violates one or more of the expressions. The node that - is most preferred is the one with the greatest sum of - weights, i.e. for each node that meets all of the scheduling - requirements (resource request, requiredDuringScheduling - anti-affinity expressions, etc.), compute a sum by iterating - through the elements of this field and adding "weight" - to the sum if the node has pods which matches the corresponding - podAffinityTerm; the node(s) with the highest sum are - the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, - a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: - description: operator represents a - key's relationship to a set of values. - Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of - string values. If the operator is - In or NotIn, the values array must - be non-empty. If the operator is - Exists or DoesNotExist, the values - array must be empty. This array - is replaced during a strategic merge - patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator - is "In", and the values array contains - only "value". The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the - pods matching the labelSelector in the specified - namespaces, where co-located is defined as - running on a node whose value of the label - with key topologyKey matches that of any node - on which any of the selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: weight associated with matching the - corresponding podAffinityTerm, in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified - by this field are not met at scheduling time, the pod - will not be scheduled onto the node. If the anti-affinity - requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod - label update), the system may or may not try to eventually - evict the pod from its node. When there are multiple - elements, the lists of nodes corresponding to each podAffinityTerm - are intersected, i.e. all terms must be satisfied. - items: - description: Defines a set of pods (namely those matching - the labelSelector relative to the given namespace(s)) - that this pod should be co-located (affinity) or not - co-located (anti-affinity) with, where co-located - is defined as running on a node whose value of the - label with key matches that of any node - on which a pod of the set of pods is running - properties: - labelSelector: - description: A label query over a set of resources, - in this case pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement - is a selector that contains values, a key, - and an operator that relates the key and - values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" - items: - type: string - type: array - topologyKey: - description: This pod should be co-located (affinity) - or not co-located (anti-affinity) with the pods - matching the labelSelector in the specified namespaces, - where co-located is defined as running on a node - whose value of the label with key topologyKey - matches that of any node on which any of the selected - pods is running. Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - args: - description: 'Optional: List of arguments' - items: - type: string - type: array - env: - description: 'Optional: List of environment variables' - items: - description: EnvVar represents an environment variable present - in a Container. - properties: - name: - description: Name of the environment variable. Must be a - C_IDENTIFIER. - type: string - value: - description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in the - container and any service environment variables. If a - variable cannot be resolved, the reference in the input - string will be unchanged. The $(VAR_NAME) syntax can be - escaped with a double $$, ie: $$(VAR_NAME). Escaped references - will never be expanded, regardless of whether the variable - exists or not. Defaults to "".' - type: string - valueFrom: - description: Source for the environment variable's value. - Cannot be used if value is not empty. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - key: - description: The key to select. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the ConfigMap or its - key must be defined - type: boolean - required: - - key - type: object - fieldRef: - description: 'Selects a field of the pod: supports metadata.name, + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - resourceFieldRef: - description: 'Selects a resource of the container: only + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' - type: string - divisor: - anyOf: + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: NVIDIA vGPU Device Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA vGPU Device Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + version: + description: NVIDIA vGPU Device Manager image tag + type: string + type: object + vgpuManager: + description: VGPUManager component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + driverManager: + description: DriverManager represents configuration for NVIDIA + Driver Manager initContainer + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: - type: integer - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: - resource - type: object - secretKeyRef: - description: Selects a key of a secret in the pod's - namespace - properties: - key: - description: The key of the secret to select from. Must - be a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: Specify whether the Secret or its key - must be defined - type: boolean - required: + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: - key - type: object - type: object - required: + type: object + type: object + required: - name - type: object - type: array - image: - pattern: '[a-zA-Z0-9\-]+' - type: string - imagePullPolicy: - description: Image pull policy - type: string - imagePullSecrets: - description: Image pull secrets - items: + type: object + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' type: string - type: array - licensingConfig: - description: 'Optional: Licensing configuration for vGPU drivers' - properties: - configMapName: + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - type: object - nodeSelector: - additionalProperties: + type: array + repository: + description: Repository represents Driver Managerrepository + path type: string - description: Node selector to control the selection of nodes (optional) - type: object - podSecurityContext: - description: 'Optional: Pod Security Context' + version: + description: Version represents NVIDIA Driver Manager image + tag(version) + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA vGPU Manager + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. properties: - fsGroup: - description: "A special supplemental group that applies to - all containers in a pod. Some volume types allow the Kubelet - to change the ownership of that volume to be owned by the - pod: \n 1. The owning GID will be the FSGroup 2. The setgid - bit is set (new files created in the volume will be owned - by FSGroup) 3. The permission bits are OR'd with rw-rw---- - \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." - format: int64 - type: integer - fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing - ownership and permission of the volume before being exposed - inside Pod. This field will only apply to volume types which - support fsGroup based ownership(and permissions). It will - have no effect on ephemeral volume types such as: secret, - configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. type: string - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in SecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in SecurityContext. If set - in both SecurityContext and PodSecurityContext, the value - specified in SecurityContext takes precedence for that container. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - SecurityContext. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence - for that container. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by the containers - in this pod. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - supplementalGroups: - description: A list of groups applied to the first process - run in each container, in addition to the container's primary - GID. If unspecified, no groups will be added to any container. - items: - format: int64 - type: integer - type: array - sysctls: - description: Sysctls hold a list of namespaced sysctls used - for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. - items: - description: Sysctl defines a kernel parameter to be set - properties: - name: - description: Name of a property to set - type: string - value: - description: Value of a property to set - type: string - required: - - name - - value - type: object - type: array - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options within a container's - SecurityContext will be used. If set in both SecurityContext - and PodSecurityContext, the value specified in SecurityContext - takes precedence. + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in + the container and any service environment variables. If + a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object type: object + required: + - name type: object - repoConfig: - description: 'Optional: Custom repo configuration for driver container' - properties: - configMapName: - type: string - destinationDir: - type: string - type: object - repository: - pattern: '[a-zA-Z0-9\.\-\/]+' + type: array + image: + description: NVIDIA vGPU Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: type: string - resources: - description: 'Optional: Define resources requests and limits for + type: array + repository: + description: NVIDIA vGPU Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each pod' - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' - type: object - type: object - securityContext: - description: 'Optional: Security Context' - properties: - allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a - process can gain more privileges than its parent process. - This bool directly controls if the no_new_privs flag will - be set on the container process. AllowPrivilegeEscalation - is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' - type: boolean - capabilities: - description: The capabilities to add/drop when running containers. - Defaults to the default set of capabilities granted by the - container runtime. - properties: - add: - description: Added capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - drop: - description: Removed capabilities - items: - description: Capability represent POSIX capabilities - type - type: string - type: array - type: object - privileged: - description: Run container in privileged mode. Processes in - privileged containers are essentially equivalent to root - on the host. Defaults to false. - type: boolean - procMount: - description: procMount denotes the type of proc mount to use - for the containers. The default is DefaultProcMount which - uses the container runtime defaults for readonly paths and - masked paths. This requires the ProcMountType feature flag - to be enabled. - type: string - readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. - Default is false. - type: boolean - runAsGroup: - description: The GID to run the entrypoint of the container - process. Uses runtime default if unset. May also be set - in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - format: int64 - type: integer - runAsNonRoot: - description: Indicates that the container must run as a non-root - user. If true, the Kubelet will validate the image at runtime - to ensure that it does not run as UID 0 (root) and fail - to start the container if it does. If unset or false, no - such validation will be performed. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: The UID to run the entrypoint of the container - process. Defaults to user specified in image metadata if - unspecified. May also be set in PodSecurityContext. If - set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. - format: int64 - type: integer - seLinuxOptions: - description: The SELinux context to be applied to the container. - If unspecified, the container runtime will allocate a random - SELinux context for each container. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. - type: string - type: object - seccompProfile: - description: The seccomp options to use by this container. - If seccomp options are provided at both the pod & container - level, the container options override the pod options. - properties: - localhostProfile: - description: localhostProfile indicates a profile defined - in a file on the node should be used. The profile must - be preconfigured on the node to work. Must be a descending - path, relative to the kubelet's configured seccomp profile - location. Must only be set if type is "Localhost". - type: string - type: - description: "type indicates which kind of seccomp profile - will be applied. Valid options are: \n Localhost - a - profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile - should be used. Unconfined - no profile should be applied." - type: string - required: - - type - type: object - windowsOptions: - description: The Windows specific settings applied to all - containers. If unspecified, the options from the PodSecurityContext - will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. - properties: - gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission - webhook (https://github.com/kubernetes-sigs/windows-gmsa) - inlines the contents of the GMSA credential spec named - by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - runAsUserName: - description: The UserName in Windows to run the entrypoint - of the container process. Defaults to the user specified - in image metadata if unspecified. May also be set in - PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext - takes precedence. - type: string - type: object - type: object - tolerations: - description: 'Optional: Set tolerations' - items: - description: The pod this Toleration is attached to tolerates - any taint that matches the triple using - the matching operator . - properties: - effect: - description: Effect indicates the taint effect to match. - Empty means match all taint effects. When specified, allowed - values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Key is the taint key that the toleration applies - to. Empty means match all taint keys. If the key is empty, - operator must be Exists; this combination means to match - all values and all keys. - type: string - operator: - description: Operator represents a key's relationship to - the value. Valid operators are Exists and Equal. Defaults - to Equal. Exists is equivalent to wildcard for value, - so that a pod can tolerate all taints of a particular - category. - type: string - tolerationSeconds: - description: TolerationSeconds represents the period of - time the toleration (which must be of effect NoExecute, - otherwise this field is ignored) tolerates the taint. - By default, it is not set, which means tolerate the taint - forever (do not evict). Zero and negative values will - be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: Value is the taint value the toleration matches - to. If the operator is Exists, the value should be empty, - otherwise just a regular string. - type: string + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - type: array - version: - pattern: '[a-zA-Z0-9\.-]+' - type: string - required: - - image - - repository - - version - type: object - required: - - dcgmExporter - - devicePlugin - - driver - - gfd - - operator - - toolkit - type: object - status: - description: ClusterPolicyStatus defines the observed state of ClusterPolicy - properties: - state: - enum: - - ignored - - ready - - notReady - type: string - required: - - state - type: object - type: object - served: true - storage: true - subresources: - status: {} + type: object + version: + description: NVIDIA vGPU Manager image tag + type: string + type: object + required: + - daemonsets + - dcgm + - dcgmExporter + - devicePlugin + - driver + - gfd + - nodeStatusExporter + - operator + - toolkit + type: object + status: + description: ClusterPolicyStatus defines the observed state of ClusterPolicy + properties: + namespace: + description: Namespace indicates a namespace in which the operator + is installed + type: string + state: + description: State indicates status of ClusterPolicy + enum: + - ignored + - ready + - notReady + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/gpu-operator-components.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/gpu-operator-components.yaml index b82e2df41a..8cbf303c77 100644 --- a/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/gpu-operator-components.yaml +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/gpu/gpu-operator-components.yaml @@ -6,281 +6,264 @@ metadata: name: gpu-operator-resources labels: app.kubernetes.io/component: "gpu-operator" - openshift.io/cluster-monitoring: "true" --- -# Source: gpu-operator/charts/node-feature-discovery/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: gpu-operator-node-feature-discovery - namespace: default + namespace: gpu-operator-resources labels: - helm.sh/chart: node-feature-discovery-2.0.0 + helm.sh/chart: node-feature-discovery-0.10.1 app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/version: "0.6.0" + app.kubernetes.io/version: "v0.10.1" app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - nodes + # when using command line flag --resource-labels to create extended resources + # you will need to uncomment "- nodes/status" + # - nodes/status + verbs: + - get + - patch + - update + - list --- -# Source: gpu-operator/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: - name: gpu-operator - namespace: default + name: gpu-operator-node-feature-discovery labels: - app.kubernetes.io/component: "gpu-operator" + helm.sh/chart: node-feature-discovery-0.8.2 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: gpu-operator + app.kubernetes.io/version: "v0.8.2" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gpu-operator-node-feature-discovery +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator-resources --- -# Source: gpu-operator/charts/node-feature-discovery/templates/configmap.yaml +# Source: gpu-operator/charts/node-feature-discovery/templates/master.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-operator-node-feature-discovery-master + namespace: gpu-operator-resources + labels: + helm.sh/chart: node-feature-discovery-0.10.1 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: gpu-operator + app.kubernetes.io/version: "v0.10.1" + app.kubernetes.io/managed-by: Helm + role: master +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: gpu-operator + role: master + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: gpu-operator + role: master + annotations: + {} + spec: + serviceAccountName: node-feature-discovery + securityContext: + {} + containers: + - name: master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "k8s.gcr.io/nfd/node-feature-discovery:v0.10.1" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: grpc + namespace: gpu-operator-resources + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-master" + resources: + {} + args: + - "--extra-label-ns=nvidia.com" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml apiVersion: v1 kind: ConfigMap metadata: - name: gpu-operator-node-feature-discovery - namespace: default + name: nfd-worker-conf + namespace: gpu-operator-resources labels: - helm.sh/chart: node-feature-discovery-2.0.0 + helm.sh/chart: node-feature-discovery-0.10.1 app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/version: "0.6.0" + app.kubernetes.io/version: "v0.10.1" app.kubernetes.io/managed-by: Helm data: - nfd-worker.conf: | + nfd-worker.conf: |- sources: pci: + deviceClassWhitelist: + - "02" + - "0200" + - "0207" + - "0300" + - "0302" deviceLabelFields: - vendor --- -# Source: gpu-operator/charts/node-feature-discovery/templates/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: gpu-operator-node-feature-discovery-master -rules: - - apiGroups: - - "" - resources: - - nodes - # when using command line flag --resource-labels to create extended resources - # you will need to uncomment "- nodes/status" - # - nodes/status - verbs: - - get - - patch - - update ---- -# Source: gpu-operator/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: gpu-operator - labels: - app.kubernetes.io/component: "gpu-operator" - -rules: - - apiGroups: - - config.openshift.io - resources: - - proxies - verbs: - - get - - apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - verbs: - - '*' - - apiGroups: - - "" - resources: - - pods - - services - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - - serviceaccounts - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - verbs: - - '*' - - apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - list - - create - - watch - - apiGroups: - - nvidia.com - resources: - - '*' - verbs: - - '*' - - apiGroups: - - scheduling.k8s.io - resources: - - priorityclasses - verbs: - - get - - list - - watch - - create - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - verbs: - - '*' - - apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list - - watch ---- -# Source: gpu-operator/charts/node-feature-discovery/templates/rbac.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: gpu-operator-node-feature-discovery-master -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: gpu-operator-node-feature-discovery-master -subjects: - - kind: ServiceAccount - name: gpu-operator-node-feature-discovery - namespace: default ---- -# Source: gpu-operator/templates/rolebinding.yaml -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: gpu-operator - labels: - app.kubernetes.io/component: "gpu-operator" - -subjects: - - kind: ServiceAccount - name: gpu-operator - namespace: default -roleRef: - kind: ClusterRole - name: gpu-operator - apiGroup: rbac.authorization.k8s.io ---- # Source: gpu-operator/charts/node-feature-discovery/templates/service.yaml apiVersion: v1 kind: Service metadata: - name: gpu-operator-node-feature-discovery - namespace: default + name: gpu-operator-node-feature-discovery-master + namespace: gpu-operator-resources labels: - helm.sh/chart: node-feature-discovery-2.0.0 + helm.sh/chart: node-feature-discovery-0.10.1 app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/version: "0.6.0" + app.kubernetes.io/version: "v0.8.2" app.kubernetes.io/managed-by: Helm + role: master spec: type: ClusterIP ports: - - name: api - port: 8080 + - port: 8080 + targetPort: grpc protocol: TCP - targetPort: api - + name: grpc + namespace: gpu-operator-resources selector: - app.kubernetes.io/component: master app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator --- -# Source: gpu-operator/charts/node-feature-discovery/templates/daemonset-worker.yaml +# Source: gpu-operator/charts/node-feature-discovery/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-feature-discovery + namespace: gpu-operator-resources + labels: + helm.sh/chart: node-feature-discovery-0.10.1 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: gpu-operator + app.kubernetes.io/version: "v0.10.1" + app.kubernetes.io/managed-by: Helm +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/worker.yaml apiVersion: apps/v1 kind: DaemonSet metadata: - name: gpu-operator-node-feature-discovery-worker - namespace: default + name: gpu-operator-node-feature-discovery-worker + namespace: gpu-operator-resources labels: - helm.sh/chart: node-feature-discovery-2.0.0 + helm.sh/chart: node-feature-discovery-0.10.1 app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/version: "0.6.0" + app.kubernetes.io/version: "v0.10.1" app.kubernetes.io/managed-by: Helm - app.kubernetes.io/component: worker + role: worker spec: selector: matchLabels: app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/component: worker + role: worker template: metadata: labels: app.kubernetes.io/name: node-feature-discovery app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/component: worker + role: worker + annotations: + {} spec: - serviceAccountName: gpu-operator-node-feature-discovery + dnsPolicy: ClusterFirstWithHostNet securityContext: {} - dnsPolicy: ClusterFirstWithHostNet containers: - - name: node-feature-discovery-master - securityContext: - {} - image: "quay.io/kubernetes_incubator/node-feature-discovery:v0.6.0" - imagePullPolicy: IfNotPresent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - command: - - "nfd-worker" - args: - - "--sleep-interval=60s" - - "--server=gpu-operator-node-feature-discovery:8080" - volumeMounts: - - name: host-boot - mountPath: "/host-boot" - readOnly: true - - name: host-os-release - mountPath: "/host-etc/os-release" - readOnly: true - - name: host-sys - mountPath: "/host-sys" - - name: source-d - mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" - - name: features-d - mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" - - name: nfd-worker-config - mountPath: "/etc/kubernetes/node-feature-discovery/" - resources: + - name: worker + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "k8s.gcr.io/nfd/node-feature-discovery:v0.8.2" + imagePullPolicy: IfNotPresent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: {} - + command: + - "nfd-worker" + args: + - "--sleep-interval=60s" + - "--server=gpu-operator-node-feature-discovery-master:8080" + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: source-d + mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" + readOnly: true + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true volumes: - name: host-boot hostPath: @@ -297,9 +280,13 @@ spec: - name: features-d hostPath: path: "/etc/kubernetes/node-feature-discovery/features.d/" - - name: nfd-worker-config + - name: nfd-worker-conf configMap: - name: gpu-operator-node-feature-discovery + name: nfd-worker-conf + namespace: gpu-operator-resources + items: + - key: nfd-worker.conf + path: nfd-worker.conf tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master @@ -310,79 +297,172 @@ spec: operator: Equal value: present --- -# Source: gpu-operator/charts/node-feature-discovery/templates/deployment-master.yaml -apiVersion: apps/v1 -kind: Deployment +# Source: gpu-operator/templates/clusterpolicy.yaml +apiVersion: nvidia.com/v1 +kind: ClusterPolicy metadata: - name: gpu-operator-node-feature-discovery-master - namespace: default + name: cluster-policy + namespace: gpu-operator-resources labels: - helm.sh/chart: node-feature-discovery-2.0.0 - app.kubernetes.io/name: node-feature-discovery - app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/version: "0.6.0" - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/component: master + app.kubernetes.io/component: "gpu-operator" + spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: node-feature-discovery - app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/component: master - template: - metadata: - labels: - app.kubernetes.io/name: node-feature-discovery - app.kubernetes.io/instance: gpu-operator - app.kubernetes.io/component: master - spec: - serviceAccountName: gpu-operator-node-feature-discovery - securityContext: - {} - containers: - - name: node-feature-discovery-master - securityContext: - {} - image: "quay.io/kubernetes_incubator/node-feature-discovery:v0.6.0" - imagePullPolicy: IfNotPresent - ports: - - name: api - containerPort: 8080 - protocol: TCP - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - command: - - "nfd-master" - args: - - --extra-label-ns=nvidia.com - resources: - {} - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - weight: 1 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Equal + operator: + defaultRuntime: docker + runtimeClass: nvidia + initContainer: + repository: nvcr.io/nvidia + image: cuda + version: 11.4.2-base-ubi8 + imagePullPolicy: IfNotPresent + daemonsets: + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + priorityClassName: system-node-critical + validator: + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator + version: v1.11.1 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + seLinuxOptions: + level: s0 + plugin: + env: + - name: WITH_WORKLOAD + value: "false" + mig: + strategy: single + psp: + enabled: false + driver: + enabled: true + repository: nvcr.io/nvidia + image: driver + version: 515.48.07 + imagePullPolicy: IfNotPresent + rdma: + enabled: false + useHostMofed: false + manager: + repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager + version: v0.4.1 + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_AUTO_DRAIN + value: "true" + - name: DRAIN_USE_FORCE + value: "false" + - name: DRAIN_POD_SELECTOR_LABEL value: "" + - name: DRAIN_TIMEOUT_SECONDS + value: 0s + - name: DRAIN_DELETE_EMPTYDIR_DATA + value: "false" + repoConfig: + configMapName: "" + certConfig: + name: "" + licensingConfig: + configMapName: "" + nlsEnabled: false + virtualTopology: + config: "" + securityContext: + privileged: true + seLinuxOptions: + level: s0 + toolkit: + enabled: true + repository: nvcr.io/nvidia/k8s + image: container-toolkit + version: v1.10.0-ubuntu20.04 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + seLinuxOptions: + level: s0 + devicePlugin: + repository: nvcr.io/nvidia + image: k8s-device-plugin + version: v0.12.2-ubi8 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: PASS_DEVICE_SPECS + value: "true" + - name: FAIL_ON_INIT_ERROR + value: "true" + - name: DEVICE_LIST_STRATEGY + value: envvar + - name: DEVICE_ID_STRATEGY + value: uuid + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: all + dcgm: + enabled: false + repository: nvcr.io/nvidia/cloud-native + image: dcgm + version: 2.4.5-1-ubuntu20.04 + imagePullPolicy: IfNotPresent + hostPort: 5555 + dcgmExporter: + repository: nvcr.io/nvidia/k8s + image: dcgm-exporter + version: 2.4.5-2.6.7-ubuntu20.04 + imagePullPolicy: IfNotPresent + env: + - name: DCGM_EXPORTER_LISTEN + value: :9400 + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: DCGM_EXPORTER_COLLECTORS + value: /etc/dcgm-exporter/dcp-metrics-included.csv + gfd: + repository: nvcr.io/nvidia + image: gpu-feature-discovery + version: v0.6.1-ubi8 + imagePullPolicy: IfNotPresent + env: + - name: GFD_SLEEP_INTERVAL + value: 60s + - name: GFD_FAIL_ON_INIT_ERROR + value: "true" + migManager: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: k8s-mig-manager + version: v0.4.2-ubuntu20.04 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: WITH_REBOOT + value: "false" + config: + name: "" + gpuClientsConfig: + name: "" + nodeStatusExporter: + enabled: false + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator + version: v1.11.1 + imagePullPolicy: IfNotPresent --- # Source: gpu-operator/templates/operator.yaml apiVersion: apps/v1 kind: Deployment metadata: name: gpu-operator - namespace: default + namespace: gpu-operator-resources labels: app.kubernetes.io/component: "gpu-operator" @@ -392,44 +472,58 @@ spec: matchLabels: app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" template: metadata: labels: app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" annotations: openshift.io/scc: restricted-readonly spec: serviceAccountName: gpu-operator + priorityClassName: system-node-critical containers: - - name: gpu-operator - image: nvcr.io/nvidia/gpu-operator:1.6.2 - imagePullPolicy: IfNotPresent - command: ["gpu-operator"] - args: - - "--zap-time-encoding=epoch" - env: - - name: WATCH_NAMESPACE - value: "" - - name: OPERATOR_NAME - value: "gpu-operator" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - volumeMounts: - - name: host-os-release - mountPath: "/host-etc/os-release" - readOnly: true - readinessProbe: - exec: - command: ["stat", "/tmp/operator-sdk-ready"] - initialDelaySeconds: 4 - periodSeconds: 10 - failureThreshold: 1 - ports: - - containerPort: 60000 - name: metrics + - name: gpu-operator + image: nvcr.io/nvidia/gpu-operator:v1.11.1 + imagePullPolicy: IfNotPresent + command: ["gpu-operator"] + args: + - --leader-elect + env: + - name: WATCH_NAMESPACE + value: "" + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 350Mi + requests: + cpu: 200m + memory: 100Mi + ports: + - name: metrics + containerPort: 8080 volumes: - name: host-os-release hostPath: @@ -437,102 +531,193 @@ spec: affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Equal value: "" --- -# Source: gpu-operator/templates/clusterpolicy.yaml -apiVersion: nvidia.com/v1 -kind: ClusterPolicy +# Source: gpu-operator/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: - name: cluster-policy - namespace: default + creationTimestamp: null + name: gpu-operator + namespace: gpu-operator-resources labels: app.kubernetes.io/component: "gpu-operator" -spec: - operator: - defaultRuntime: containerd - validator: - repository: nvcr.io/nvidia/k8s - image: cuda-sample - version: vectoradd-cuda10.2 - imagePullPolicy: IfNotPresent - driver: - repository: nvcr.io/nvidia - image: driver - version: 510.47.03 - imagePullPolicy: Always - repoConfig: - configMapName: "" - destinationDir: "" - licensingConfig: - configMapName: "" - tolerations: - - effect: NoSchedule - key: nvidia.com/gpu - operator: Exists - nodeSelector: - nvidia.com/gpu.present: "true" - securityContext: - privileged: true - seLinuxOptions: - level: s0 - toolkit: - repository: nvcr.io/nvidia/k8s - image: container-toolkit - version: 1.4.7-ubuntu18.04 - imagePullPolicy: IfNotPresent - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: nvidia.com/gpu - operator: Exists - nodeSelector: - nvidia.com/gpu.present: "true" - securityContext: - privileged: true - seLinuxOptions: - level: s0 - devicePlugin: - repository: nvcr.io/nvidia - image: k8s-device-plugin - version: v0.8.2-ubi8 - imagePullPolicy: IfNotPresent - nodeSelector: - nvidia.com/gpu.present: "true" - securityContext: - privileged: true - args: - - --mig-strategy=single - - --pass-device-specs=true - - --fail-on-init-error=true - - --device-list-strategy=envvar - - --nvidia-driver-root=/run/nvidia/driver - dcgmExporter: - repository: nvcr.io/nvidia/k8s - image: dcgm-exporter - version: 2.1.4-2.2.0-ubuntu20.04 - imagePullPolicy: IfNotPresent - args: - - -f - - /etc/dcgm-exporter/dcp-metrics-included.csv - gfd: - repository: nvcr.io/nvidia - image: gpu-feature-discovery - version: v0.4.1 - imagePullPolicy: IfNotPresent - nodeSelector: - nvidia.com/gpu.present: "true" - migStrategy: single - discoveryIntervalSeconds: 60 +rules: +- apiGroups: + - config.openshift.io + resources: + - proxies + verbs: + - get +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + - clusterroles + - clusterrolebindings + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + - serviceaccounts + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - create + - watch + - update +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - list + - create + - watch + - update +- apiGroups: + - nvidia.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch + - create +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - '*' +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - gpu-operator-restricted +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - create + - get + - update + - list +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list + - watch +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - create + - update + - watch +- apiGroups: + - image.openshift.io + resources: + - imagestreams + verbs: + - get + - list + - watch +--- +# Source: gpu-operator/templates/rolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator + labels: + app.kubernetes.io/component: "gpu-operator" + +subjects: +- kind: ServiceAccount + name: gpu-operator + namespace: gpu-operator-resources +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator-resources +roleRef: + kind: ClusterRole + name: gpu-operator + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-operator + namespace: gpu-operator-resources + labels: + app.kubernetes.io/component: "gpu-operator" diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 97c17fd7eb..9987fe8312 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -45,6 +45,9 @@ const ( CNIAddonVersion = "VPC_ADDON_VERSION" CorednsAddonVersion = "COREDNS_ADDON_VERSION" GcWorkloadPath = "GC_WORKLOAD" + KubeproxyAddonVersion = "KUBE_PROXY_ADDON_VERSION" + EnableIPv6 = "ENABLE_IPv6" + IPFamily = "IP_FAMILY" AwsNodeMachineType = "AWS_NODE_MACHINE_TYPE" AwsAvailabilityZone1 = "AWS_AVAILABILITY_ZONE_1" AwsAvailabilityZone2 = "AWS_AVAILABILITY_ZONE_2" diff --git a/test/e2e/shared/template.go b/test/e2e/shared/template.go index efe08932bb..89cb49635a 100644 --- a/test/e2e/shared/template.go +++ b/test/e2e/shared/template.go @@ -198,6 +198,13 @@ func getBootstrapTemplate(e2eCtx *E2EContext) *cfn_bootstrap.Template { // ApplyTemplate will render a cluster template and apply it to the management cluster. func ApplyTemplate(ctx context.Context, configCluster clusterctl.ConfigClusterInput, clusterProxy framework.ClusterProxy) error { + workloadClusterTemplate := GetTemplate(ctx, configCluster) + Byf("Applying the %s cluster template yaml to the cluster", configCluster.Flavor) + return clusterProxy.Apply(ctx, workloadClusterTemplate) +} + +// GetTemplate will render a cluster template. +func GetTemplate(ctx context.Context, configCluster clusterctl.ConfigClusterInput) []byte { Expect(ctx).NotTo(BeNil(), "ctx is required for ApplyClusterTemplateAndWait") Byf("Getting the cluster template yaml") @@ -215,6 +222,5 @@ func ApplyTemplate(ctx context.Context, configCluster clusterctl.ConfigClusterIn }) Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") - Byf("Applying the %s cluster template yaml to the cluster", configCluster.Flavor) - return clusterProxy.Apply(ctx, workloadClusterTemplate) + return workloadClusterTemplate } diff --git a/test/e2e/suites/managed/aws_node_env.go b/test/e2e/suites/managed/aws_node_env.go new file mode 100644 index 0000000000..8b7c6d698a --- /dev/null +++ b/test/e2e/suites/managed/aws_node_env.go @@ -0,0 +1,89 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package managed + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws/client" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-aws/test/e2e/shared" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" +) + +type UpdateAwsNodeVersionSpecInput struct { + E2EConfig *clusterctl.E2EConfig + BootstrapClusterProxy framework.ClusterProxy + AWSSession client.ConfigProvider + Namespace *corev1.Namespace + ClusterName string +} + +// CheckAwsNodeEnvVarsSet implements a test for setting environment variables on the `aws-node` DaemonSet. +func CheckAwsNodeEnvVarsSet(ctx context.Context, inputGetter func() UpdateAwsNodeVersionSpecInput) { + input := inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil") + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil") + Expect(input.AWSSession).ToNot(BeNil(), "Invalid argument. input.AWSSession can't be nil") + Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil") + Expect(input.ClusterName).ShouldNot(HaveLen(0), "Invalid argument. input.ClusterName can't be empty") + + mgmtClient := input.BootstrapClusterProxy.GetClient() + controlPlaneName := getControlPlaneName(input.ClusterName) + + shared.Byf("Getting control plane: %s", controlPlaneName) + controlPlane := &ekscontrolplanev1.AWSManagedControlPlane{} + err := mgmtClient.Get(ctx, crclient.ObjectKey{Namespace: input.Namespace.Name, Name: controlPlaneName}, controlPlane) + Expect(err).ToNot(HaveOccurred()) + + shared.Byf("Checking environment variables are set on AWSManagedControlPlane: %s", controlPlaneName) + Expect(controlPlane.Spec.VpcCni.Env).NotTo(BeNil()) + Expect(len(controlPlane.Spec.VpcCni.Env)).Should(BeNumerically(">", 1)) + + shared.Byf("Checking if aws-node has been updated with the defined environment variables on the workload cluster") + daemonSet := &appsv1.DaemonSet{} + + clusterClient := e2eCtx.Environment.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName).GetClient() + err = clusterClient.Get(ctx, crclient.ObjectKey{Namespace: "kube-system", Name: "aws-node"}, daemonSet) + Expect(err).ToNot(HaveOccurred()) + + for _, container := range daemonSet.Spec.Template.Spec.Containers { + if container.Name == "aws-node" { + Expect(matchEnvVar(container.Env, corev1.EnvVar{Name: "FOO", Value: "BAR"})).Should(BeTrue()) + Expect(matchEnvVar(container.Env, corev1.EnvVar{Name: "ENABLE_PREFIX_DELEGATION", Value: "true"})).Should(BeTrue()) + break + } + } +} + +func matchEnvVar(s []corev1.EnvVar, ev corev1.EnvVar) bool { + for _, e := range s { + if e == ev { + return true + } + } + return false +} diff --git a/test/e2e/suites/managed/cluster.go b/test/e2e/suites/managed/cluster.go index 74427ef70d..6e0e64f481 100644 --- a/test/e2e/suites/managed/cluster.go +++ b/test/e2e/suites/managed/cluster.go @@ -83,14 +83,14 @@ func ManagedClusterSpec(ctx context.Context, inputGetter func() ManagedClusterSp shared.Byf("Checking EKS cluster is active") eksClusterName := getEKSClusterName(input.Namespace.Name, input.ClusterName) - verifyClusterActiveAndOwned(eksClusterName, input.ClusterName, input.AWSSession) + verifyClusterActiveAndOwned(eksClusterName, input.AWSSession) if input.CluserSpecificRoles { ginkgo.By("Checking that the cluster specific IAM role exists") - VerifyRoleExistsAndOwned(fmt.Sprintf("%s-iam-service-role", input.ClusterName), input.ClusterName, true, input.AWSSession) + verifyRoleExistsAndOwned(fmt.Sprintf("%s-iam-service-role", input.ClusterName), eksClusterName, true, input.AWSSession) } else { ginkgo.By("Checking that the cluster default IAM role exists") - VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, input.ClusterName, false, input.AWSSession) + verifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, eksClusterName, false, input.AWSSession) } shared.Byf("Checking kubeconfig secrets exist") diff --git a/test/e2e/suites/managed/eks_test.go b/test/e2e/suites/managed/eks_test.go index b517546b37..60ad846b1a 100644 --- a/test/e2e/suites/managed/eks_test.go +++ b/test/e2e/suites/managed/eks_test.go @@ -38,7 +38,7 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() { var ( namespace *corev1.Namespace ctx context.Context - specName = "eks-nodes" + specName = "cluster" clusterName string cniAddonName = "vpc-cni" corednsAddonName = "coredns" @@ -57,7 +57,7 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() { clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) ginkgo.By("default iam role should exist") - VerifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, clusterName, false, e2eCtx.BootstrapUserAWSSession) + verifyRoleExistsAndOwned(ekscontrolplanev1.DefaultEKSControlPlaneRole, eksClusterName, false, e2eCtx.BootstrapUserAWSSession) ginkgo.By("should create an EKS control plane") ManagedClusterSpec(ctx, func() ManagedClusterSpecInput { @@ -74,6 +74,17 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() { } }) + ginkgo.By("should set environment variables on the aws-node daemonset") + CheckAwsNodeEnvVarsSet(ctx, func() UpdateAwsNodeVersionSpecInput { + return UpdateAwsNodeVersionSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + } + }) + ginkgo.By("should have the VPC CNI installed") CheckAddonExistsSpec(ctx, func() CheckAddonExistsSpecInput { return CheckAddonExistsSpecInput{ @@ -128,6 +139,23 @@ var _ = ginkgo.Describe("[managed] [general] EKS cluster tests", func() { } }) + ginkgo.By("should create a managed node pool with launch template and scale") + ManagedMachinePoolSpec(ctx, func() ManagedMachinePoolSpecInput { + return ManagedMachinePoolSpecInput{ + E2EConfig: e2eCtx.E2EConfig, + ConfigClusterFn: defaultConfigCluster, + BootstrapClusterProxy: e2eCtx.Environment.BootstrapClusterProxy, + AWSSession: e2eCtx.BootstrapUserAWSSession, + Namespace: namespace, + ClusterName: clusterName, + IncludeScaling: true, + Cleanup: true, + Flavor: EKSManagedPoolWithLaunchTemplateOnlyFlavor, + UsesLaunchTemplate: true, + EKSKubernetesVersion: eksKubernetesVersion, + } + }) + shared.Byf("getting cluster with name %s", clusterName) cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ Getter: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), diff --git a/test/e2e/suites/managed/helpers.go b/test/e2e/suites/managed/helpers.go index 73f7e61b0b..a797afb731 100644 --- a/test/e2e/suites/managed/helpers.go +++ b/test/e2e/suites/managed/helpers.go @@ -38,15 +38,20 @@ import ( // EKS related constants. const ( - EKSManagedPoolFlavor = "eks-managedmachinepool" - EKSControlPlaneOnlyFlavor = "eks-control-plane-only" - EKSControlPlaneOnlyWithAddonFlavor = "eks-control-plane-only-withaddon" - EKSMachineDeployOnlyFlavor = "eks-machine-deployment-only" - EKSManagedPoolOnlyFlavor = "eks-managed-machinepool-only" + EKSManagedPoolFlavor = "eks-managedmachinepool" + EKSControlPlaneOnlyFlavor = "eks-control-plane-only" + EKSControlPlaneOnlyWithAddonFlavor = "eks-control-plane-only-withaddon" + EKSMachineDeployOnlyFlavor = "eks-machine-deployment-only" + EKSManagedPoolOnlyFlavor = "eks-managed-machinepool-only" + EKSManagedPoolWithLaunchTemplateOnlyFlavor = "eks-managed-machinepool-with-launch-template-only" ) type DefaultConfigClusterFn func(clusterName, namespace string) clusterctl.ConfigClusterInput +func getEKSNodegroupWithLaunchTemplateName(namespace, clusterName string) string { + return fmt.Sprintf("%s_%s-pool-lt-0", namespace, clusterName) +} + func getEKSClusterName(namespace, clusterName string) string { return fmt.Sprintf("%s_%s-control-plane", namespace, clusterName) } @@ -59,11 +64,11 @@ func getControlPlaneName(clusterName string) string { return fmt.Sprintf("%s-control-plane", clusterName) } -func verifyClusterActiveAndOwned(eksClusterName, clusterName string, sess client.ConfigProvider) { +func verifyClusterActiveAndOwned(eksClusterName string, sess client.ConfigProvider) { cluster, err := getEKSCluster(eksClusterName, sess) Expect(err).NotTo(HaveOccurred()) - tagName := infrav1.ClusterTagKey(clusterName) + tagName := infrav1.ClusterTagKey(eksClusterName) tagValue, ok := cluster.Tags[tagName] Expect(ok).To(BeTrue(), "expecting the cluster owned tag to exist") Expect(*tagValue).To(BeEquivalentTo(string(infrav1.ResourceLifecycleOwned))) @@ -110,7 +115,7 @@ func verifyConfigMapExists(ctx context.Context, name, namespace string, k8sclien Expect(err).ShouldNot(HaveOccurred()) } -func VerifyRoleExistsAndOwned(roleName string, clusterName string, checkOwned bool, sess client.ConfigProvider) { +func verifyRoleExistsAndOwned(roleName string, eksClusterName string, checkOwned bool, sess client.ConfigProvider) { iamClient := iam.New(sess) input := &iam.GetRoleInput{ RoleName: aws.String(roleName), @@ -121,7 +126,7 @@ func VerifyRoleExistsAndOwned(roleName string, clusterName string, checkOwned bo if checkOwned { found := false - expectedTagName := infrav1.ClusterAWSCloudProviderTagKey(clusterName) + expectedTagName := infrav1.ClusterAWSCloudProviderTagKey(eksClusterName) for _, tag := range output.Role.Tags { if *tag.Key == expectedTagName && *tag.Value == string(infrav1.ResourceLifecycleOwned) { found = true diff --git a/test/e2e/suites/managed/machine_pool.go b/test/e2e/suites/managed/machine_pool.go index f1d74f28a6..1782f7bb2b 100644 --- a/test/e2e/suites/managed/machine_pool.go +++ b/test/e2e/suites/managed/machine_pool.go @@ -43,6 +43,8 @@ type ManagedMachinePoolSpecInput struct { ClusterName string IncludeScaling bool Cleanup bool + UsesLaunchTemplate bool + EKSKubernetesVersion string } // ManagedMachinePoolSpec implements a test for creating a managed machine pool. @@ -67,7 +69,21 @@ func ManagedMachinePoolSpec(ctx context.Context, inputGetter func() ManagedMachi configCluster := input.ConfigClusterFn(input.ClusterName, input.Namespace.Name) configCluster.Flavor = EKSManagedPoolOnlyFlavor configCluster.WorkerMachineCount = pointer.Int64Ptr(1) - err := shared.ApplyTemplate(ctx, configCluster, input.BootstrapClusterProxy) + workloadClusterTemplate := shared.GetTemplate(ctx, configCluster) + if input.UsesLaunchTemplate { + userDataTemplate := `#!/bin/bash +/etc/eks/bootstrap.sh %s \ + --container-runtime containerd +` + eksClusterName := getEKSClusterName(input.Namespace.Name, input.ClusterName) + userData := fmt.Sprintf(userDataTemplate, eksClusterName) + userDataEncoded := base64.StdEncoding.EncodeToString([]byte(userData)) + workloadClusterTemplate = []byte(strings.ReplaceAll(string(workloadClusterTemplate), "USER_DATA", userDataEncoded)) + workloadClusterTemplate = []byte(strings.ReplaceAll(string(workloadClusterTemplate), "EKS_KUBERNETES_VERSION", input.EKSKubernetesVersion)) + } + shared.Byf(string(workloadClusterTemplate)) + shared.Byf("Applying the %s cluster template yaml to the cluster", configCluster.Flavor) + err := input.BootstrapClusterProxy.Apply(ctx, workloadClusterTemplate) Expect(err).ShouldNot(HaveOccurred()) shared.Byf("Waiting for the machine pool to be running") @@ -79,9 +95,20 @@ func ManagedMachinePoolSpec(ctx context.Context, inputGetter func() ManagedMachi Expect(len(mp)).To(Equal(1)) shared.Byf("Check the status of the node group") - nodeGroupName := getEKSNodegroupName(input.Namespace.Name, input.ClusterName) + var nodeGroupName string + if input.UsesLaunchTemplate { + nodeGroupName = getEKSNodegroupWithLaunchTemplateName(input.Namespace.Name, input.ClusterName) + } else { + nodeGroupName = getEKSNodegroupName(input.Namespace.Name, input.ClusterName) + } eksClusterName := getEKSClusterName(input.Namespace.Name, input.ClusterName) - verifyManagedNodeGroup(eksClusterName, nodeGroupName, true, input.AWSSession) + if input.ManagedMachinePool { + nodeGroupName := getEKSNodegroupName(input.Namespace.Name, input.ClusterName) + verifyManagedNodeGroup(eksClusterName, nodeGroupName, true, input.AWSSession) + } else { + asgName := getASGName(input.ClusterName) + verifyASG(eksClusterName, asgName, true, input.AWSSession) + } if input.IncludeScaling { // TODO (richardcase): should this be a separate spec? ginkgo.By("Scaling the machine pool up") diff --git a/util/system/util.go b/util/system/util.go index 390573b420..cffff0735f 100644 --- a/util/system/util.go +++ b/util/system/util.go @@ -18,6 +18,7 @@ package system import ( "fmt" + "github.com/aws/aws-sdk-go/aws/endpoints" "os" "path/filepath" @@ -68,3 +69,19 @@ func GetNamespaceFromFile(nsFilePath string) (string, error) { } return string(namespace), nil } + +// GetPartitionFromRegion returns the cluster partition. +func GetPartitionFromRegion(region string) string { + switch region { + case endpoints.UsGovEast1RegionID, endpoints.UsGovWest1RegionID: + return endpoints.AwsUsGovPartitionID + case endpoints.CnNorth1RegionID, endpoints.CnNorthwest1RegionID: + return endpoints.AwsCnPartitionID + case endpoints.UsIsoEast1RegionID, endpoints.UsIsoWest1RegionID: + return endpoints.AwsIsoPartitionID + case endpoints.UsIsobEast1RegionID: + return endpoints.AwsIsoBPartitionID + default: + return endpoints.AwsPartitionID + } +}