From 1cb7e547cafecd9c4e9d8349126162ce1c543044 Mon Sep 17 00:00:00 2001 From: Vince Prignano Date: Thu, 1 Nov 2018 07:37:29 -0700 Subject: [PATCH] Node join Signed-off-by: Vince Prignano --- Gopkg.lock | 13 ++ Makefile | 7 +- .../examples/aws/machines.yaml.template | 32 +-- ...der_v1alpha1_awsclusterproviderconfig.yaml | 8 + config/manager/manager.yaml | 2 + .../awsclusterproviderconfig_types.go | 6 + .../awsclusterproviderstatus_types.go | 6 - pkg/apis/awsprovider/v1alpha1/register.go | 19 ++ .../v1alpha1/zz_generated.deepcopy.go | 20 +- pkg/cloud/aws/actuators/cluster/BUILD | 2 +- pkg/cloud/aws/actuators/cluster/actuator.go | 90 ++------ .../aws/actuators/cluster/actuator_test.go | 4 + pkg/cloud/aws/actuators/machine/BUILD | 25 +- pkg/cloud/aws/actuators/machine/actuator.go | 41 +++- .../aws/actuators/machine/actuator_test.go | 160 ------------- .../{mock.go => machineinterface.go} | 28 ++- pkg/cloud/aws/services/ec2/BUILD | 1 + pkg/cloud/aws/services/ec2/bastion.go | 18 +- pkg/cloud/aws/services/ec2/instances.go | 132 +++++------ pkg/cloud/aws/services/ec2/instances_test.go | 26 ++- pkg/cloud/aws/services/interfaces.go | 3 +- pkg/cloud/aws/services/mocks/ec2interface.go | 21 +- pkg/cloud/aws/services/userdata/BUILD.bazel | 14 ++ pkg/cloud/aws/services/userdata/bastion.go | 42 ++++ .../aws/services/userdata/controlplane.go | 76 +++++++ pkg/cloud/aws/services/userdata/node.go | 69 ++++++ pkg/cloud/aws/services/userdata/userdata.go | 63 ++++++ pkg/deployer/BUILD.bazel | 16 ++ pkg/deployer/deployer.go | 103 +++++++++ pkg/tokens/BUILD.bazel | 17 ++ pkg/tokens/tokens.go | 91 ++++++++ .../.github/PULL_REQUEST_TEMPLATE.md | 2 + .../k8s.io/cluster-bootstrap/CONTRIBUTING.md | 7 + .../cluster-bootstrap/Godeps/Godeps.json | 150 ++++++++++++ vendor/k8s.io/cluster-bootstrap/Godeps/OWNERS | 2 + vendor/k8s.io/cluster-bootstrap/Godeps/Readme | 5 + vendor/k8s.io/cluster-bootstrap/LICENSE | 202 +++++++++++++++++ vendor/k8s.io/cluster-bootstrap/OWNERS | 6 + vendor/k8s.io/cluster-bootstrap/README.md | 21 ++ .../cluster-bootstrap/SECURITY_CONTACTS | 17 ++ .../cluster-bootstrap/code-of-conduct.md | 3 + vendor/k8s.io/cluster-bootstrap/token/OWNERS | 5 + .../cluster-bootstrap/token/api/BUILD.bazel | 13 ++ .../k8s.io/cluster-bootstrap/token/api/doc.go | 20 ++ .../cluster-bootstrap/token/api/types.go | 112 +++++++++ .../cluster-bootstrap/token/util/BUILD.bazel | 19 ++ .../cluster-bootstrap/token/util/helpers.go | 133 +++++++++++ .../token/util/helpers_test.go | 213 ++++++++++++++++++ 48 files changed, 1674 insertions(+), 411 deletions(-) delete mode 100644 pkg/cloud/aws/actuators/machine/actuator_test.go rename pkg/cloud/aws/actuators/machine/mock_machineiface/{mock.go => machineinterface.go} (91%) mode change 100644 => 100755 create mode 100644 pkg/cloud/aws/services/userdata/BUILD.bazel create mode 100644 pkg/cloud/aws/services/userdata/bastion.go create mode 100644 pkg/cloud/aws/services/userdata/controlplane.go create mode 100644 pkg/cloud/aws/services/userdata/node.go create mode 100644 pkg/cloud/aws/services/userdata/userdata.go create mode 100644 pkg/deployer/BUILD.bazel create mode 100644 pkg/deployer/deployer.go create mode 100644 pkg/tokens/BUILD.bazel create mode 100644 pkg/tokens/tokens.go create mode 100644 vendor/k8s.io/cluster-bootstrap/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 vendor/k8s.io/cluster-bootstrap/CONTRIBUTING.md create mode 100644 vendor/k8s.io/cluster-bootstrap/Godeps/Godeps.json create mode 100644 vendor/k8s.io/cluster-bootstrap/Godeps/OWNERS create mode 100644 vendor/k8s.io/cluster-bootstrap/Godeps/Readme create mode 100644 vendor/k8s.io/cluster-bootstrap/LICENSE create mode 100644 vendor/k8s.io/cluster-bootstrap/OWNERS create mode 100644 vendor/k8s.io/cluster-bootstrap/README.md create mode 100644 vendor/k8s.io/cluster-bootstrap/SECURITY_CONTACTS create mode 100644 vendor/k8s.io/cluster-bootstrap/code-of-conduct.md create mode 100644 vendor/k8s.io/cluster-bootstrap/token/OWNERS create mode 100644 vendor/k8s.io/cluster-bootstrap/token/api/BUILD.bazel create mode 100644 vendor/k8s.io/cluster-bootstrap/token/api/doc.go create mode 100644 vendor/k8s.io/cluster-bootstrap/token/api/types.go create mode 100644 vendor/k8s.io/cluster-bootstrap/token/util/BUILD.bazel create mode 100644 vendor/k8s.io/cluster-bootstrap/token/util/helpers.go create mode 100644 vendor/k8s.io/cluster-bootstrap/token/util/helpers_test.go diff --git a/Gopkg.lock b/Gopkg.lock index fb20be40f4..a023cf9a92 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -765,6 +765,17 @@ revision = "1f13a808da65775f22cbf47862c4e5898d8f4ca1" version = "kubernetes-1.11.2" +[[projects]] + branch = "master" + digest = "1:a260d2283fb2b612c237df5a7de12133a5c83063a6e16955199bff86e3e4269e" + name = "k8s.io/cluster-bootstrap" + packages = [ + "token/api", + "token/util", + ] + pruneopts = "" + revision = "f574069107f7246cf0a827d06126404028914e97" + [[projects]] branch = "master" digest = "1:bc58db16947c92752bf6d1ead0d2596bd638456a2c23e00c567505680d772670" @@ -941,6 +952,8 @@ "k8s.io/client-go/tools/clientcmd", "k8s.io/client-go/tools/clientcmd/api", "k8s.io/client-go/tools/record", + "k8s.io/cluster-bootstrap/token/api", + "k8s.io/cluster-bootstrap/token/util", "k8s.io/code-generator/cmd/deepcopy-gen", "sigs.k8s.io/cluster-api/cmd/clusterctl/cmd", "sigs.k8s.io/cluster-api/pkg/apis", diff --git a/Makefile b/Makefile index b334ec6f02..3c642c7249 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ test: generate ## Run tests bazel test --nosandbox_debug //pkg/... //cmd/... $(BAZEL_ARGS) .PHONY: copy-genmocks -copy-genmocks: test ## Copies generated mocks into the repository +copy-genmocks: ## Copies generated mocks into the repository cp -Rf bazel-genfiles/pkg/* pkg/ BAZEL_DOCKER_ARGS_COMMON := --define=MANAGER_IMAGE_NAME=$(MANAGER_IMAGE_NAME) --define=MANAGER_IMAGE_TAG=$(MANAGER_IMAGE_TAG) $(BAZEL_ARGS) @@ -152,7 +152,7 @@ manifests-dev: ## Push development manifest .PHONY: create-cluster create-cluster: ## Create a Kubernetes cluster on AWS using examples - clusterctl create cluster -v3 --provider aws -m ./cmd/clusterctl/examples/aws/out/machines.yaml -c ./cmd/clusterctl/examples/aws/out/cluster.yaml -p ./cmd/clusterctl/examples/aws/out/provider-components.yaml + clusterctl create cluster -v 4 --provider aws -m ./cmd/clusterctl/examples/aws/out/machines.yaml -c ./cmd/clusterctl/examples/aws/out/cluster.yaml -p ./cmd/clusterctl/examples/aws/out/provider-components.yaml -a ./cmd/clusterctl/examples/aws/out/addons.yaml lint-full: dep-ensure ## Run slower linters to detect possible issues bazel run //:lint-full $(BAZEL_ARGS) @@ -162,6 +162,9 @@ ifneq ($(FASTBUILD),y) ## Define slow dependency targets here +reset-bazel: ## Deep cleaning for bazel + bazel clean --expunge + generate: dep-ensure ## Run go generate GOPATH=$(shell go env GOPATH) bazel run //:generate $(BAZEL_ARGS) $(MAKE) dep-ensure diff --git a/cmd/clusterctl/examples/aws/machines.yaml.template b/cmd/clusterctl/examples/aws/machines.yaml.template index d4434e2182..d8bd6ab0e8 100644 --- a/cmd/clusterctl/examples/aws/machines.yaml.template +++ b/cmd/clusterctl/examples/aws/machines.yaml.template @@ -16,19 +16,19 @@ items: instanceType: "${CONTROL_PLANE_MACHINE_TYPE}" iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" keyName: "${SSH_KEY_NAME}" - # - apiVersion: "cluster.k8s.io/v1alpha1" - # kind: Machine - # metadata: - # generateName: aws-node- - # labels: - # set: node - # spec: - # versions: - # kubelet: v1.12.0 - # providerConfig: - # value: - # apiVersion: awsprovider/v1alpha1 - # kind: AWSMachineProviderConfig - # instanceType: "${NODE_MACHINE_TYPE}" - # iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io" - # keyName: "${SSH_KEY_NAME}" + - apiVersion: "cluster.k8s.io/v1alpha1" + kind: Machine + metadata: + generateName: aws-node- + labels: + set: node + spec: + versions: + kubelet: v1.12.0 + providerConfig: + value: + apiVersion: awsprovider/v1alpha1 + kind: AWSMachineProviderConfig + instanceType: "${NODE_MACHINE_TYPE}" + iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io" + keyName: "${SSH_KEY_NAME}" diff --git a/config/crds/awsprovider_v1alpha1_awsclusterproviderconfig.yaml b/config/crds/awsprovider_v1alpha1_awsclusterproviderconfig.yaml index 5fff145b98..5d3114358d 100644 --- a/config/crds/awsprovider_v1alpha1_awsclusterproviderconfig.yaml +++ b/config/crds/awsprovider_v1alpha1_awsclusterproviderconfig.yaml @@ -16,6 +16,12 @@ spec: properties: apiVersion: type: string + caCertificate: + format: byte + type: string + caKey: + format: byte + type: string kind: type: string metadata: @@ -24,6 +30,8 @@ spec: type: string sshKeyName: type: string + required: + - caKey version: v1alpha1 status: acceptedNames: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 82c8f1f2e6..596ccbf13b 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -54,7 +54,9 @@ spec: containers: - name: manager image: SET_BY_PATCH + imagePullPolicy: Always args: + - "-v=3" - "-logtostderr=true" - "-stderrthreshold=INFO" volumeMounts: diff --git a/pkg/apis/awsprovider/v1alpha1/awsclusterproviderconfig_types.go b/pkg/apis/awsprovider/v1alpha1/awsclusterproviderconfig_types.go index 879883da3a..ff3de3d607 100644 --- a/pkg/apis/awsprovider/v1alpha1/awsclusterproviderconfig_types.go +++ b/pkg/apis/awsprovider/v1alpha1/awsclusterproviderconfig_types.go @@ -35,6 +35,12 @@ type AWSClusterProviderConfig struct { // SSHKeyName is the name of the ssh key to attach to the bastion host. SSHKeyName string `json:"sshKeyName,omitempty"` + + // CACertificate is a PEM encoded CA Certificate for the control plane nodes. + CACertificate []byte `json:"caCertificate,omitempty"` + + // CAPrivateKey is a PEM encoded PKCS1 CA PrivateKey for the control plane nodes. + CAPrivateKey []byte `json:"caKey,omitemptuy"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/awsprovider/v1alpha1/awsclusterproviderstatus_types.go b/pkg/apis/awsprovider/v1alpha1/awsclusterproviderstatus_types.go index 5609a8f970..ce91e30f8f 100644 --- a/pkg/apis/awsprovider/v1alpha1/awsclusterproviderstatus_types.go +++ b/pkg/apis/awsprovider/v1alpha1/awsclusterproviderstatus_types.go @@ -33,12 +33,6 @@ type AWSClusterProviderStatus struct { Region string `json:"region,omitempty"` Network Network `json:"network,omitempty"` Bastion Instance `json:"bastion,omitempty"` - - // CACertificate is a PEM encoded CA Certificate for the control plane nodes. - CACertificate []byte - - // CAPrivateKey is a PEM encoded PKCS1 CA PrivateKey for the control plane nodes. - CAPrivateKey []byte } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/awsprovider/v1alpha1/register.go b/pkg/apis/awsprovider/v1alpha1/register.go index 22fa22fbbc..5d26845e0d 100644 --- a/pkg/apis/awsprovider/v1alpha1/register.go +++ b/pkg/apis/awsprovider/v1alpha1/register.go @@ -129,3 +129,22 @@ func EncodeClusterStatus(status *AWSClusterProviderStatus) (*runtime.RawExtensio Raw: rawBytes, }, nil } + +// EncodeClusterConfig marshals the cluster config. +func EncodeClusterConfig(status *AWSClusterProviderConfig) (*runtime.RawExtension, error) { + if status == nil { + return &runtime.RawExtension{}, nil + } + + var rawBytes []byte + var err error + + // TODO: use apimachinery conversion https://godoc.org/k8s.io/apimachinery/pkg/runtime#Convert_runtime_Object_To_runtime_RawExtension + if rawBytes, err = json.Marshal(status); err != nil { + return nil, err + } + + return &runtime.RawExtension{ + Raw: rawBytes, + }, nil +} diff --git a/pkg/apis/awsprovider/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/awsprovider/v1alpha1/zz_generated.deepcopy.go index ffeeab4ea2..00416df2da 100644 --- a/pkg/apis/awsprovider/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/awsprovider/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,16 @@ func (in *AWSClusterProviderConfig) DeepCopyInto(out *AWSClusterProviderConfig) *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.CACertificate != nil { + in, out := &in.CACertificate, &out.CACertificate + *out = make([]byte, len(*in)) + copy(*out, *in) + } + if in.CAPrivateKey != nil { + in, out := &in.CAPrivateKey, &out.CAPrivateKey + *out = make([]byte, len(*in)) + copy(*out, *in) + } return } @@ -53,16 +63,6 @@ func (in *AWSClusterProviderStatus) DeepCopyInto(out *AWSClusterProviderStatus) in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Network.DeepCopyInto(&out.Network) in.Bastion.DeepCopyInto(&out.Bastion) - if in.CACertificate != nil { - in, out := &in.CACertificate, &out.CACertificate - *out = make([]byte, len(*in)) - copy(*out, *in) - } - if in.CAPrivateKey != nil { - in, out := &in.CAPrivateKey, &out.CAPrivateKey - *out = make([]byte, len(*in)) - copy(*out, *in) - } return } diff --git a/pkg/cloud/aws/actuators/cluster/BUILD b/pkg/cloud/aws/actuators/cluster/BUILD index 1f8f7bdd64..a828e32d91 100644 --- a/pkg/cloud/aws/actuators/cluster/BUILD +++ b/pkg/cloud/aws/actuators/cluster/BUILD @@ -11,13 +11,13 @@ go_library( "//pkg/cloud/aws/services/certificates:go_default_library", "//pkg/cloud/aws/services/ec2:go_default_library", "//pkg/cloud/aws/services/elb:go_default_library", + "//pkg/deployer:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", - "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/controller/error:go_default_library", diff --git a/pkg/cloud/aws/actuators/cluster/actuator.go b/pkg/cloud/aws/actuators/cluster/actuator.go index 4c2a831691..e2fccafdd3 100644 --- a/pkg/cloud/aws/actuators/cluster/actuator.go +++ b/pkg/cloud/aws/actuators/cluster/actuator.go @@ -14,20 +14,18 @@ package cluster import ( - "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/client-go/tools/clientcmd" providerv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1" service "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services" "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/certificates" ec2svc "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/ec2" elbsvc "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/elb" + "sigs.k8s.io/cluster-api-provider-aws/pkg/deployer" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" controllerError "sigs.k8s.io/cluster-api/pkg/controller/error" @@ -35,6 +33,8 @@ import ( // Actuator is responsible for performing cluster reconciliation type Actuator struct { + *deployer.Deployer + clustersGetter client.ClustersGetter servicesGetter service.Getter } @@ -56,6 +56,7 @@ func NewActuator(params ActuatorParams) *Actuator { res.servicesGetter = new(defaultServicesGetter) } + res.Deployer = deployer.New(res.servicesGetter) return res } @@ -76,6 +77,10 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) (reterr error) { } defer func() { + if err := a.storeClusterConfig(cluster, config); err != nil { + glog.Errorf("failed to store provider config for cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err) + } + if err := a.storeClusterStatus(cluster, status); err != nil { glog.Errorf("failed to store provider status for cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err) } @@ -84,13 +89,14 @@ func (a *Actuator) Reconcile(cluster *clusterv1.Cluster) (reterr error) { // Store some config parameters in the status. status.Region = config.Region - if len(status.CACertificate) == 0 { + if len(config.CACertificate) == 0 { caCert, caKey, err := certificates.NewCertificateAuthority() if err != nil { return errors.Wrap(err, "Failed to generate a CA for the control plane") } - status.CACertificate = certificates.EncodeCertPEM(caCert) - status.CAPrivateKey = certificates.EncodePrivateKeyPEM(caKey) + + config.CACertificate = certificates.EncodeCertPEM(caCert) + config.CAPrivateKey = certificates.EncodePrivateKeyPEM(caKey) } // Create new aws session. @@ -163,73 +169,21 @@ func (a *Actuator) Delete(cluster *clusterv1.Cluster) error { return nil } -// GetIP returns the IP of a machine, but this is going away. -func (a *Actuator) GetIP(cluster *clusterv1.Cluster, _ *clusterv1.Machine) (string, error) { - if cluster.Status.ProviderStatus != nil { - - // Load provider status. - status, err := providerv1.ClusterStatusFromProviderStatus(cluster.Status.ProviderStatus) - if err != nil { - return "", errors.Errorf("failed to load cluster provider status: %v", err) - } - - if status.Network.APIServerELB.DNSName != "" { - return status.Network.APIServerELB.DNSName, nil - } - } - - // Load provider config. - config, err := providerv1.ClusterConfigFromProviderConfig(cluster.Spec.ProviderConfig) - if err != nil { - return "", errors.Errorf("failed to load cluster provider config: %v", err) - } - - sess := a.servicesGetter.Session(config) - elb := a.servicesGetter.ELB(sess) - return elb.GetAPIServerDNSName(cluster.Name) -} - -// GetKubeConfig returns the kubeconfig after the bootstrap process is complete. -func (a *Actuator) GetKubeConfig(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (string, error) { - - // Load provider status. - status, err := providerv1.ClusterStatusFromProviderStatus(cluster.Status.ProviderStatus) - if err != nil { - return "", errors.Errorf("failed to load cluster provider status: %v", err) - } - - cert, err := certificates.DecodeCertPEM(status.CACertificate) - if err != nil { - return "", errors.Wrap(err, "failed to decode CA Cert") - } else if cert == nil { - return "", errors.New("certificate not found in status") - } - - key, err := certificates.DecodePrivateKeyPEM(status.CAPrivateKey) - if err != nil { - return "", errors.Wrap(err, "failed to decode private key") - } else if key == nil { - return "", errors.New("key not found in status") - } +func (a *Actuator) storeClusterConfig(cluster *clusterv1.Cluster, config *providerv1.AWSClusterProviderConfig) error { + clusterClient := a.clustersGetter.Clusters(cluster.Namespace) - dnsName, err := a.GetIP(cluster, machine) + ext, err := providerv1.EncodeClusterConfig(config) if err != nil { - return "", errors.Wrap(err, "failed to get DNS address") + return err } - server := fmt.Sprintf("https://%s:6443", dnsName) + cluster.Spec.ProviderConfig.Value = ext - cfg, err := certificates.NewKubeconfig(server, cert, key) - if err != nil { - return "", errors.Wrap(err, "failed to generate a kubeconfig") + if _, err := clusterClient.Update(cluster); err != nil { + return err } - yaml, err := clientcmd.Write(*cfg) - if err != nil { - return "", errors.Wrap(err, "failed to serialize config to yaml") - } - - return string(yaml), nil + return nil } func (a *Actuator) storeClusterStatus(cluster *clusterv1.Cluster, status *providerv1.AWSClusterProviderStatus) error { @@ -237,13 +191,13 @@ func (a *Actuator) storeClusterStatus(cluster *clusterv1.Cluster, status *provid ext, err := providerv1.EncodeClusterStatus(status) if err != nil { - return fmt.Errorf("failed to update cluster status for cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err) + return err } cluster.Status.ProviderStatus = ext if _, err := clusterClient.UpdateStatus(cluster); err != nil { - return fmt.Errorf("failed to update cluster status for cluster %q in namespace %q: %v", cluster.Name, cluster.Namespace, err) + return err } return nil diff --git a/pkg/cloud/aws/actuators/cluster/actuator_test.go b/pkg/cloud/aws/actuators/cluster/actuator_test.go index 8b6047359e..d2e5399fae 100644 --- a/pkg/cloud/aws/actuators/cluster/actuator_test.go +++ b/pkg/cloud/aws/actuators/cluster/actuator_test.go @@ -189,6 +189,10 @@ func TestReconcile(t *testing.T) { }, } + clusters.ci.EXPECT(). + Update(gomock.AssignableToTypeOf(cluster)). + Return(cluster, nil) + clusters.ci.EXPECT(). UpdateStatus(gomock.AssignableToTypeOf(cluster)). Return(cluster, nil) diff --git a/pkg/cloud/aws/actuators/machine/BUILD b/pkg/cloud/aws/actuators/machine/BUILD index ee2d877f22..9c5e40b01b 100644 --- a/pkg/cloud/aws/actuators/machine/BUILD +++ b/pkg/cloud/aws/actuators/machine/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", @@ -15,32 +15,19 @@ go_library( "//pkg/cloud/aws/services:go_default_library", "//pkg/cloud/aws/services/ec2:go_default_library", "//pkg/cloud/aws/services/elb:go_default_library", + "//pkg/deployer:go_default_library", + "//pkg/tokens:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", + "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library", "//vendor/sigs.k8s.io/cluster-api/pkg/controller/error:go_default_library", ], ) - -go_test( - name = "go_default_test", - srcs = ["actuator_test.go"], - embed = [":go_default_library"], - deps = [ - "//pkg/apis/awsprovider/v1alpha1:go_default_library", - "//pkg/cloud/aws/actuators/machine/mock_machineiface:go_default_library", - "//pkg/cloud/aws/services:go_default_library", - "//pkg/cloudtest:go_default_library", - "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", - "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", - "//vendor/github.com/golang/mock/gomock:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1:go_default_library", - "//vendor/sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1:go_default_library", - ], -) diff --git a/pkg/cloud/aws/actuators/machine/actuator.go b/pkg/cloud/aws/actuators/machine/actuator.go index 958e320f1f..b5ecbf5b4a 100644 --- a/pkg/cloud/aws/actuators/machine/actuator.go +++ b/pkg/cloud/aws/actuators/machine/actuator.go @@ -24,10 +24,15 @@ import ( "github.com/aws/aws-sdk-go/service/elb" "github.com/golang/glog" "github.com/pkg/errors" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1" service "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services" ec2svc "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/ec2" elbsvc "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/elb" + "sigs.k8s.io/cluster-api-provider-aws/pkg/deployer" + "sigs.k8s.io/cluster-api-provider-aws/pkg/tokens" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" controllerError "sigs.k8s.io/cluster-api/pkg/controller/error" @@ -35,6 +40,8 @@ import ( // Actuator is responsible for performing machine reconciliation. type Actuator struct { + *deployer.Deployer + machinesGetter client.MachinesGetter servicesGetter service.Getter } @@ -56,6 +63,7 @@ func NewActuator(params ActuatorParams) *Actuator { res.servicesGetter = new(defaultServicesGetter) } + res.Deployer = deployer.New(res.servicesGetter) return res } @@ -107,7 +115,38 @@ func (a *Actuator) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine return errors.Wrap(err, "failed to get machine config") } - i, err := a.ec2(clusterConfig).CreateOrGetMachine(machine, status, config, clusterStatus, cluster) + controlPlaneURL, err := a.GetIP(cluster, nil) + if err != nil { + return errors.Wrap(err, "failed to retrieve controlplane url during machine creation") + } + + var bootstrapToken string + if machine.ObjectMeta.Labels["set"] == "node" { + kubeConfig, err := a.GetKubeConfig(cluster, nil) + if err != nil { + return errors.Wrap(err, "failed to retrieve kubeconfig during machine creation") + } + + clientConfig, err := clientcmd.BuildConfigFromKubeconfigGetter(controlPlaneURL, func() (*clientcmdapi.Config, error) { + return clientcmd.Load([]byte(kubeConfig)) + }) + + if err != nil { + return errors.Wrap(err, "failed to retrieve kubeconfig during machine creation") + } + + coreClient, err := corev1.NewForConfig(clientConfig) + if err != nil { + return errors.Wrap(err, "failed to initialize new corev1 client") + } + + bootstrapToken, err = tokens.NewBootstrap(coreClient, 10*time.Minute) + if err != nil { + return errors.Wrap(err, "failed to create new bootstrap token") + } + } + + i, err := a.ec2(clusterConfig).CreateOrGetMachine(machine, status, config, clusterStatus, clusterConfig, cluster, bootstrapToken) if err != nil { if ec2svc.IsFailedDependency(errors.Cause(err)) { diff --git a/pkg/cloud/aws/actuators/machine/actuator_test.go b/pkg/cloud/aws/actuators/machine/actuator_test.go deleted file mode 100644 index f9bc6bad12..0000000000 --- a/pkg/cloud/aws/actuators/machine/actuator_test.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright © 2018 The Kubernetes Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package machine_test - -import ( - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/golang/mock/gomock" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1" - "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/actuators/machine" - "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/actuators/machine/mock_machineiface" - service "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services" - "sigs.k8s.io/cluster-api-provider-aws/pkg/cloudtest" - clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" - client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" -) - -type ec2svc struct { - instance *v1alpha1.Instance -} - -func (m *ec2svc) InstanceIfExists(instanceID *string) (*v1alpha1.Instance, error) { return nil, nil } -func (m *ec2svc) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, cluster *clusterv1.Cluster) (*v1alpha1.Instance, error) { - return nil, nil -} -func (m *ec2svc) TerminateInstance(instanceID string) error { return nil } -func (m *ec2svc) DeleteBastion(instanceID string, status *v1alpha1.AWSClusterProviderStatus) error { - return nil -} -func (m *ec2svc) CreateOrGetMachine(machine *clusterv1.Machine, status *v1alpha1.AWSMachineProviderStatus, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, cluster *clusterv1.Cluster) (*v1alpha1.Instance, error) { - return m.instance, nil -} -func (m *ec2svc) UpdateInstanceSecurityGroups(instanceID string, securityGroups []string) error { - return nil -} -func (m *ec2svc) UpdateResourceTags(resourceID *string, create map[string]string, remove map[string]string) error { - return nil -} -func (m *ec2svc) ReconcileNetwork(clusterName string, network *v1alpha1.Network) error { return nil } -func (m *ec2svc) ReconcileBastion(clusterName, keyName string, status *v1alpha1.AWSClusterProviderStatus) error { - return nil -} -func (m *ec2svc) DeleteNetwork(clusterName string, network *v1alpha1.Network) error { return nil } - -type machinesvc struct { - mock *mock_machineiface.MockMachineInterface -} - -func (m *machinesvc) Machines(namespace string) client.MachineInterface { - return m.mock -} - -type getter struct { - ec2 service.EC2Interface -} - -func (g *getter) Session(*v1alpha1.AWSClusterProviderConfig) *session.Session { - return nil -} -func (g *getter) EC2(*session.Session) service.EC2Interface { - return g.ec2 -} -func (g *getter) ELB(*session.Session) service.ELBInterface { - return nil -} - -func TestCreate(t *testing.T) { - testcases := []struct { - name string - machine *clusterv1.Machine - service *ec2svc - machineExpects func(*mock_machineiface.MockMachineInterface) - }{ - { - name: "ensure machine and status are both updated", - machine: &clusterv1.Machine{ - Spec: clusterv1.MachineSpec{ - ProviderConfig: clusterv1.ProviderConfig{ - Value: cloudtest.RuntimeRawExtension(t, v1alpha1.AWSMachineProviderConfig{}), - }, - }, - Status: clusterv1.MachineStatus{}, - }, - service: &ec2svc{ - instance: &v1alpha1.Instance{ - ID: "hello world", - }, - }, - machineExpects: func(mock *mock_machineiface.MockMachineInterface) { - mock.EXPECT().Update(&clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "cluster-api-provider-aws": "true", - }, - }, - Spec: clusterv1.MachineSpec{ - ProviderConfig: clusterv1.ProviderConfig{ - Value: cloudtest.RuntimeRawExtension(t, v1alpha1.AWSMachineProviderConfig{}), - }, - }, - }).Return(nil, nil) - - mock.EXPECT().UpdateStatus(&clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "cluster-api-provider-aws": "true", - }, - }, - Spec: clusterv1.MachineSpec{ - ProviderConfig: clusterv1.ProviderConfig{ - Value: cloudtest.RuntimeRawExtension(t, v1alpha1.AWSMachineProviderConfig{}), - }, - }, - Status: clusterv1.MachineStatus{ - ProviderStatus: cloudtest.RuntimeRawExtension(t, v1alpha1.AWSMachineProviderStatus{ - InstanceID: aws.String("hello world"), - InstanceState: aws.String(""), - }), - }, - }).Return(nil, nil) - }, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - mockMachine := mock_machineiface.NewMockMachineInterface(mockCtrl) - tc.machineExpects(mockMachine) - - a := machine.NewActuator(machine.ActuatorParams{ - ServicesGetter: &getter{ - ec2: tc.service, - }, - MachinesGetter: &machinesvc{ - mock: mockMachine, - }, - }) - err := a.Create(&clusterv1.Cluster{}, tc.machine) - if err != nil { - t.Fatalf("did not expect an error: %v", err) - } - }) - } -} diff --git a/pkg/cloud/aws/actuators/machine/mock_machineiface/mock.go b/pkg/cloud/aws/actuators/machine/mock_machineiface/machineinterface.go old mode 100644 new mode 100755 similarity index 91% rename from pkg/cloud/aws/actuators/machine/mock_machineiface/mock.go rename to pkg/cloud/aws/actuators/machine/mock_machineiface/machineinterface.go index 5cbbefa27f..3df38a8d2f --- a/pkg/cloud/aws/actuators/machine/mock_machineiface/mock.go +++ b/pkg/cloud/aws/actuators/machine/mock_machineiface/machineinterface.go @@ -1,15 +1,19 @@ -// Copyright © 2018 The Kubernetes Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by MockGen. DO NOT EDIT. // Source: sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1 (interfaces: MachineInterface) diff --git a/pkg/cloud/aws/services/ec2/BUILD b/pkg/cloud/aws/services/ec2/BUILD index 0a88685541..819dc11a6a 100644 --- a/pkg/cloud/aws/services/ec2/BUILD +++ b/pkg/cloud/aws/services/ec2/BUILD @@ -26,6 +26,7 @@ go_library( deps = [ "//pkg/apis/awsprovider/v1alpha1:go_default_library", "//pkg/cloud/aws/services/awserrors:go_default_library", + "//pkg/cloud/aws/services/userdata:go_default_library", "//pkg/cloud/aws/services/wait:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library", diff --git a/pkg/cloud/aws/services/ec2/bastion.go b/pkg/cloud/aws/services/ec2/bastion.go index 9f85530540..9b4ecce983 100644 --- a/pkg/cloud/aws/services/ec2/bastion.go +++ b/pkg/cloud/aws/services/ec2/bastion.go @@ -22,23 +22,10 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/userdata" ) const ( - bastionUserData = `#!/bin/bash - -BASTION_BOOTSTRAP_FILE=bastion_bootstrap.sh -BASTION_BOOTSTRAP=https://s3.amazonaws.com/aws-quickstart/quickstart-linux-bastion/scripts/bastion_bootstrap.sh - -curl -s -o $BASTION_BOOTSTRAP_FILE $BASTION_BOOTSTRAP -chmod +x $BASTION_BOOTSTRAP_FILE - -# This gets us far enough in the bastion script to be useful. -apt-get -y update && apt-get -y install python-pip -pip install --upgrade pip &> /dev/null - -./$BASTION_BOOTSTRAP_FILE --banner https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}scripts/banner_message.txt --enable true -` defaultSSHKeyName = "default" ) @@ -131,13 +118,14 @@ func (s *Service) describeBastionInstance(clusterName string, status *v1alpha1.A func (s *Service) getDefaultBastion(clusterName string, region string, network v1alpha1.Network, keyName string) *v1alpha1.Instance { name := fmt.Sprintf("%s-bastion", clusterName) tags := s.buildTags(clusterName, ResourceLifecycleOwned, name, TagValueBastionRole, nil) + userData, _ := userdata.NewBastion(&userdata.BastionInput{}) i := &v1alpha1.Instance{ Type: "t2.micro", SubnetID: network.Subnets.FilterPublic()[0].ID, ImageID: s.defaultBastionAMILookup(region), KeyName: aws.String(keyName), - UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(bastionUserData))), + UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(userData))), SecurityGroupIDs: []string{ network.SecurityGroups[v1alpha1.SecurityGroupBastion].ID, }, diff --git a/pkg/cloud/aws/services/ec2/instances.go b/pkg/cloud/aws/services/ec2/instances.go index 80914a7ea7..83104ecb6b 100644 --- a/pkg/cloud/aws/services/ec2/instances.go +++ b/pkg/cloud/aws/services/ec2/instances.go @@ -15,13 +15,13 @@ package ec2 import ( "encoding/base64" - "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/glog" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsprovider/v1alpha1" + "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/services/userdata" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" ) @@ -63,6 +63,7 @@ func (s *Service) InstanceIfExists(instanceID *string) (*v1alpha1.Instance, erro input := &ec2.DescribeInstancesInput{ InstanceIds: []*string{instanceID}, + Filters: []*ec2.Filter{s.filterInstanceStates(ec2.InstanceStateNamePending, ec2.InstanceStateNameRunning)}, } out, err := s.EC2.DescribeInstances(input) @@ -81,8 +82,8 @@ func (s *Service) InstanceIfExists(instanceID *string) (*v1alpha1.Instance, erro } // CreateInstance runs an ec2 instance. -func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, cluster *clusterv1.Cluster) (*v1alpha1.Instance, error) { - glog.V(2).Info("Attempting to create an instance") +func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, clusterConfig *v1alpha1.AWSClusterProviderConfig, cluster *clusterv1.Cluster, bootstrapToken string) (*v1alpha1.Instance, error) { + glog.V(2).Infof("Creating a new instance for machine %q", machine.Name) input := &v1alpha1.Instance{ Type: config.InstanceType, @@ -110,7 +111,7 @@ func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AW sns := clusterStatus.Network.Subnets.FilterPrivate() if len(sns) == 0 { return nil, NewFailedDependency( - errors.New("failed to run instance, no subnets available"), + errors.Errorf("failed to run machine %q, no subnets available", machine.Name), ) } input.SubnetID = sns[0].ID @@ -121,31 +122,50 @@ func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AW if clusterStatus.Network.SecurityGroups[v1alpha1.SecurityGroupControlPlane] == nil { return nil, NewFailedDependency( - errors.New("failed to run control plane, security group not available"), + errors.New("failed to run controlplane, security group not available"), ) } - if len(clusterStatus.CACertificate) == 0 { - return input, errors.New("Cluster Provider Status is missing CACertificate") + if len(clusterConfig.CACertificate) == 0 { + return nil, errors.New("failed to run controlplane, missing CACertificate") } - if len(clusterStatus.CAPrivateKey) == 0 { - return input, errors.New("Cluster Provider Status is missing CAPrivateKey") + if len(clusterConfig.CAPrivateKey) == 0 { + return nil, errors.New("failed to run controlplane, missing CAPrivateKey") } - input.UserData = aws.String(initControlPlaneScript(clusterStatus.CACertificate, - clusterStatus.CAPrivateKey, - clusterStatus.Network.APIServerELB.DNSName, - cluster.Name, - cluster.Spec.ClusterNetwork.ServiceDomain, - cluster.Spec.ClusterNetwork.Pods.CIDRBlocks[0], - cluster.Spec.ClusterNetwork.Services.CIDRBlocks[0], - machine.Spec.Versions.ControlPlane)) + userData, err := userdata.NewControlPlane(&userdata.ControlPlaneInput{ + CACert: string(clusterConfig.CACertificate), + CAKey: string(clusterConfig.CAPrivateKey), + ELBAddress: clusterStatus.Network.APIServerELB.DNSName, + ClusterName: cluster.Name, + PodSubnet: cluster.Spec.ClusterNetwork.Pods.CIDRBlocks[0], + ServiceSubnet: cluster.Spec.ClusterNetwork.Services.CIDRBlocks[0], + ServiceDomain: cluster.Spec.ClusterNetwork.ServiceDomain, + KubernetesVersion: machine.Spec.Versions.ControlPlane, + }) + + if err != nil { + return input, err + } + input.UserData = aws.String(userData) input.SecurityGroupIDs = append(input.SecurityGroupIDs, clusterStatus.Network.SecurityGroups[v1alpha1.SecurityGroupControlPlane].ID) } if machine.ObjectMeta.Labels["set"] == "node" { input.SecurityGroupIDs = append(input.SecurityGroupIDs, clusterStatus.Network.SecurityGroups[v1alpha1.SecurityGroupNode].ID) + + userData, err := userdata.NewNode(&userdata.NodeInput{ + CACert: string(clusterConfig.CACertificate), + BootstrapToken: bootstrapToken, + ELBAddress: clusterStatus.Network.APIServerELB.DNSName, + }) + + if err != nil { + return input, err + } + + input.UserData = aws.String(userData) } // Pick SSH key, if any. @@ -161,17 +181,17 @@ func (s *Service) CreateInstance(machine *clusterv1.Machine, config *v1alpha1.AW // TerminateInstance terminates an EC2 instance. // Returns nil on success, error in all other cases. func (s *Service) TerminateInstance(instanceID string) error { - glog.V(2).Infof("Attempting to terminate instance %q", instanceID) + glog.V(2).Infof("Attempting to terminate instance with id %q", instanceID) input := &ec2.TerminateInstancesInput{ InstanceIds: aws.StringSlice([]string{instanceID}), } if _, err := s.EC2.TerminateInstances(input); err != nil { - return errors.Wrapf(err, "failed to terminate instance %q", instanceID) + return errors.Wrapf(err, "failed to terminate instance with id %q", instanceID) } - glog.V(2).Infof("termination request sent for EC2 instance %q", instanceID) + glog.V(2).Infof("Terminated instance with id %q", instanceID) return nil } @@ -182,7 +202,7 @@ func (s *Service) TerminateInstanceAndWait(instanceID string) error { return err } - glog.V(2).Infof("waiting for EC2 instance %q to terminate", instanceID) + glog.V(2).Infof("Waiting for EC2 instance with id %q to terminate", instanceID) input := &ec2.DescribeInstancesInput{ InstanceIds: aws.StringSlice([]string{instanceID}), @@ -196,39 +216,30 @@ func (s *Service) TerminateInstanceAndWait(instanceID string) error { } // CreateOrGetMachine will either return an existing instance or create and return an instance. -func (s *Service) CreateOrGetMachine(machine *clusterv1.Machine, status *v1alpha1.AWSMachineProviderStatus, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, cluster *clusterv1.Cluster) (*v1alpha1.Instance, error) { - glog.V(2).Info("Attempting to create or get machine") +func (s *Service) CreateOrGetMachine(machine *clusterv1.Machine, status *v1alpha1.AWSMachineProviderStatus, config *v1alpha1.AWSMachineProviderConfig, clusterStatus *v1alpha1.AWSClusterProviderStatus, clusterConfig *v1alpha1.AWSClusterProviderConfig, cluster *clusterv1.Cluster, bootstrapToken string) (*v1alpha1.Instance, error) { + glog.V(2).Infof("Attempting to create or get machine %q", machine.Name) // instance id exists, try to get it if status.InstanceID != nil { - glog.V(2).Infof("Looking up instance %q", *status.InstanceID) - instance, err := s.InstanceIfExists(status.InstanceID) + glog.V(2).Infof("Looking up machine %q by id %q", machine.Name, *status.InstanceID) - // if there was no error, return the found instance - if err == nil { + instance, err := s.InstanceIfExists(status.InstanceID) + if err != nil && !IsNotFound(err) { + return nil, errors.Wrapf(err, "failed to look up machine %q by id %q", machine.Name, *status.InstanceID) + } else if err == nil && instance != nil { return instance, nil } - - // if there was an error but it's not IsNotFound then it's a real error - if !IsNotFound(err) { - return instance, errors.Wrapf(err, "instance %q was not found", *status.InstanceID) - } - - return instance, errors.Wrapf(err, "failed to look up instance %q", *status.InstanceID) } - glog.V(2).Infof("Looking up instance by tags") + glog.V(2).Infof("Looking up machine %q by tags", machine.Name) instance, err := s.InstanceByTags(machine, cluster) if err != nil && !IsNotFound(err) { - return instance, errors.Wrap(err, "failed to query instance by tags") - } - if err == nil && instance != nil { + return nil, errors.Wrapf(err, "failed to query machine %q instance by tags", machine.Name) + } else if err == nil && instance != nil { return instance, nil } - // otherwise let's create it - glog.V(2).Info("Instance did not exist, attempting to creating it") - return s.CreateInstance(machine, config, clusterStatus, cluster) + return s.CreateInstance(machine, config, clusterStatus, clusterConfig, cluster, bootstrapToken) } func (s *Service) runInstance(i *v1alpha1.Instance) (*v1alpha1.Instance, error) { @@ -378,42 +389,3 @@ func fromSDKTypeToInstance(v *ec2.Instance) *v1alpha1.Instance { return i } - -// initControlPlaneScript returns the b64 encoded script to run on start up. -// The cert must be CertPEM encoded and the key must be PrivateKeyPEM encoded -// TODO: convert to using cloud-init module rather than a startup script -func initControlPlaneScript(caCert, caKey []byte, elbDNSName, clusterName, dnsDomain, podSubnet, serviceSubnet, k8sVersion string) string { - // The script must start with #!. If it goes on the next line Dedent will start the script with a \n. - return fmt.Sprintf(`#!/usr/bin/env bash - -mkdir -p /etc/kubernetes/pki - -echo '%s' > /etc/kubernetes/pki/ca.crt -echo '%s' > /etc/kubernetes/pki/ca.key - -PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) - -cat >/tmp/kubeadm.yaml < /dev/null + +./$BASTION_BOOTSTRAP_FILE --enable true +` +) + +// BastionInput defines the context to generate a bastion instance user data. +type BastionInput struct { + baseUserData +} + +// NewBastion returns the user data string to be used on a bastion instance. +func NewBastion(input *BastionInput) (string, error) { + input.Header = defaultHeader + return generate("bastion", bastionBashScript, input) +} diff --git a/pkg/cloud/aws/services/userdata/controlplane.go b/pkg/cloud/aws/services/userdata/controlplane.go new file mode 100644 index 0000000000..558d440ef2 --- /dev/null +++ b/pkg/cloud/aws/services/userdata/controlplane.go @@ -0,0 +1,76 @@ +// Copyright © 2018 The Kubernetes Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package userdata + +const ( + controlPlaneBashScript = `{{.Header}} + +mkdir -p /etc/kubernetes/pki + +echo '{{.CACert}}' > /etc/kubernetes/pki/ca.crt +echo '{{.CAKey}}' > /etc/kubernetes/pki/ca.key + +PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) +HOSTNAME="$(hostname -f 2>/dev/null || curl http://169.254.169.254/latest/meta-data/local-hostname)" + +cat >/tmp/kubeadm.yaml </tmp/cluster-info.yaml </tmp/kubeadm-node.yaml <`. This is the prefix to be used before the + // token ID. + BootstrapTokenSecretPrefix = "bootstrap-token-" + + // SecretTypeBootstrapToken is used during the automated bootstrap process (first + // implemented by kubeadm). It stores tokens that are used to sign well known + // ConfigMaps. They may also eventually be used for authentication. + SecretTypeBootstrapToken v1.SecretType = "bootstrap.kubernetes.io/token" + + // BootstrapTokenIDKey is the id of this token. This can be transmitted in the + // clear and encoded in the name of the secret. It must be a random 6 character + // string that matches the regexp `^([a-z0-9]{6})$`. Required. + BootstrapTokenIDKey = "token-id" + + // BootstrapTokenSecretKey is the actual secret. It must be a random 16 character + // string that matches the regexp `^([a-z0-9]{16})$`. Required. + BootstrapTokenSecretKey = "token-secret" + + // BootstrapTokenExpirationKey is when this token should be expired and no + // longer used. A controller will delete this resource after this time. This + // is an absolute UTC time using RFC3339. If this cannot be parsed, the token + // should be considered invalid. Optional. + BootstrapTokenExpirationKey = "expiration" + + // BootstrapTokenDescriptionKey is a description in human-readable format that + // describes what the bootstrap token is used for. Optional. + BootstrapTokenDescriptionKey = "description" + + // BootstrapTokenExtraGroupsKey is a comma-separated list of group names. + // The bootstrap token will authenticate as these groups in addition to the + // "system:bootstrappers" group. + BootstrapTokenExtraGroupsKey = "auth-extra-groups" + + // BootstrapTokenUsagePrefix is the prefix for the other usage constants that specifies different + // functions of a bootstrap token + BootstrapTokenUsagePrefix = "usage-bootstrap-" + + // BootstrapTokenUsageSigningKey signals that this token should be used to + // sign configs as part of the bootstrap process. Value must be "true". Any + // other value is assumed to be false. Optional. + BootstrapTokenUsageSigningKey = "usage-bootstrap-signing" + + // BootstrapTokenUsageAuthentication signals that this token should be used + // as a bearer token to authenticate against the Kubernetes API. The bearer + // token takes the form "." and authenticates as the + // user "system:bootstrap:" in the "system:bootstrappers" group + // as well as any groups specified using BootstrapTokenExtraGroupsKey. + // Value must be "true". Any other value is assumed to be false. Optional. + BootstrapTokenUsageAuthentication = "usage-bootstrap-authentication" + + // ConfigMapClusterInfo defines the name for the ConfigMap where the information how to connect and trust the cluster exist + ConfigMapClusterInfo = "cluster-info" + + // KubeConfigKey defines at which key in the Data object of the ConfigMap the KubeConfig object is stored + KubeConfigKey = "kubeconfig" + + // JWSSignatureKeyPrefix defines what key prefix the JWS-signed tokens have + JWSSignatureKeyPrefix = "jws-kubeconfig-" + + // BootstrapUserPrefix is the username prefix bootstrapping bearer tokens + // authenticate as. The full username given is "system:bootstrap:". + BootstrapUserPrefix = "system:bootstrap:" + + // BootstrapDefaultGroup is the default group for bootstrapping bearer + // tokens (in addition to any groups from BootstrapTokenExtraGroupsKey). + BootstrapDefaultGroup = "system:bootstrappers" + + // BootstrapGroupPattern is the valid regex pattern that all groups + // assigned to a bootstrap token by BootstrapTokenExtraGroupsKey must match. + // See also util.ValidateBootstrapGroupName() + BootstrapGroupPattern = `\Asystem:bootstrappers:[a-z0-9:-]{0,255}[a-z0-9]\z` + + // BootstrapTokenPattern defines the {id}.{secret} regular expression pattern + BootstrapTokenPattern = `\A([a-z0-9]{6})\.([a-z0-9]{16})\z` + + // BootstrapTokenIDPattern defines token's id regular expression pattern + BootstrapTokenIDPattern = `\A([a-z0-9]{6})\z` + + // BootstrapTokenIDBytes defines the number of bytes used for the Bootstrap Token's ID field + BootstrapTokenIDBytes = 6 + + // BootstrapTokenSecretBytes defines the number of bytes used the Bootstrap Token's Secret field + BootstrapTokenSecretBytes = 16 +) + +// KnownTokenUsages specifies the known functions a token will get. +var KnownTokenUsages = []string{"signing", "authentication"} diff --git a/vendor/k8s.io/cluster-bootstrap/token/util/BUILD.bazel b/vendor/k8s.io/cluster-bootstrap/token/util/BUILD.bazel new file mode 100644 index 0000000000..483cc9ace2 --- /dev/null +++ b/vendor/k8s.io/cluster-bootstrap/token/util/BUILD.bazel @@ -0,0 +1,19 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["helpers.go"], + importmap = "sigs.k8s.io/cluster-api-provider-aws/vendor/k8s.io/cluster-bootstrap/token/util", + importpath = "k8s.io/cluster-bootstrap/token/util", + visibility = ["//visibility:public"], + deps = [ + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/cluster-bootstrap/token/api:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["helpers_test.go"], + embed = [":go_default_library"], +) diff --git a/vendor/k8s.io/cluster-bootstrap/token/util/helpers.go b/vendor/k8s.io/cluster-bootstrap/token/util/helpers.go new file mode 100644 index 0000000000..5565cb26d8 --- /dev/null +++ b/vendor/k8s.io/cluster-bootstrap/token/util/helpers.go @@ -0,0 +1,133 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bufio" + "crypto/rand" + "fmt" + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cluster-bootstrap/token/api" +) + +// validBootstrapTokenChars defines the characters a bootstrap token can consist of +const validBootstrapTokenChars = "0123456789abcdefghijklmnopqrstuvwxyz" + +var ( + // BootstrapTokenRegexp is a compiled regular expression of TokenRegexpString + BootstrapTokenRegexp = regexp.MustCompile(api.BootstrapTokenPattern) + // BootstrapTokenIDRegexp is a compiled regular expression of TokenIDRegexpString + BootstrapTokenIDRegexp = regexp.MustCompile(api.BootstrapTokenIDPattern) + // BootstrapGroupRegexp is a compiled regular expression of BootstrapGroupPattern + BootstrapGroupRegexp = regexp.MustCompile(api.BootstrapGroupPattern) +) + +// GenerateBootstrapToken generates a new, random Bootstrap Token. +func GenerateBootstrapToken() (string, error) { + tokenID, err := randBytes(api.BootstrapTokenIDBytes) + if err != nil { + return "", err + } + + tokenSecret, err := randBytes(api.BootstrapTokenSecretBytes) + if err != nil { + return "", err + } + + return TokenFromIDAndSecret(tokenID, tokenSecret), nil +} + +// randBytes returns a random string consisting of the characters in +// validBootstrapTokenChars, with the length customized by the parameter +func randBytes(length int) (string, error) { + // len("0123456789abcdefghijklmnopqrstuvwxyz") = 36 which doesn't evenly divide + // the possible values of a byte: 256 mod 36 = 4. Discard any random bytes we + // read that are >= 252 so the bytes we evenly divide the character set. + const maxByteValue = 252 + + var ( + b byte + err error + token = make([]byte, length) + ) + + reader := bufio.NewReaderSize(rand.Reader, length*2) + for i := range token { + for { + if b, err = reader.ReadByte(); err != nil { + return "", err + } + if b < maxByteValue { + break + } + } + + token[i] = validBootstrapTokenChars[int(b)%len(validBootstrapTokenChars)] + } + + return string(token), nil +} + +// TokenFromIDAndSecret returns the full token which is of the form "{id}.{secret}" +func TokenFromIDAndSecret(id, secret string) string { + return fmt.Sprintf("%s.%s", id, secret) +} + +// IsValidBootstrapToken returns whether the given string is valid as a Bootstrap Token and +// in other words satisfies the BootstrapTokenRegexp +func IsValidBootstrapToken(token string) bool { + return BootstrapTokenRegexp.MatchString(token) +} + +// IsValidBootstrapTokenID returns whether the given string is valid as a Bootstrap Token ID and +// in other words satisfies the BootstrapTokenIDRegexp +func IsValidBootstrapTokenID(tokenID string) bool { + return BootstrapTokenIDRegexp.MatchString(tokenID) +} + +// BootstrapTokenSecretName returns the expected name for the Secret storing the +// Bootstrap Token in the Kubernetes API. +func BootstrapTokenSecretName(tokenID string) string { + return fmt.Sprintf("%s%s", api.BootstrapTokenSecretPrefix, tokenID) +} + +// ValidateBootstrapGroupName checks if the provided group name is a valid +// bootstrap group name. Returns nil if valid or a validation error if invalid. +func ValidateBootstrapGroupName(name string) error { + if BootstrapGroupRegexp.Match([]byte(name)) { + return nil + } + return fmt.Errorf("bootstrap group %q is invalid (must match %s)", name, api.BootstrapGroupPattern) +} + +// ValidateUsages validates that the passed in string are valid usage strings for bootstrap tokens. +func ValidateUsages(usages []string) error { + validUsages := sets.NewString(api.KnownTokenUsages...) + invalidUsages := sets.NewString() + for _, usage := range usages { + if !validUsages.Has(usage) { + invalidUsages.Insert(usage) + } + } + if len(invalidUsages) > 0 { + return fmt.Errorf("invalid bootstrap token usage string: %s, valid usage options: %s", strings.Join(invalidUsages.List(), ","), strings.Join(api.KnownTokenUsages, ",")) + } + return nil +} diff --git a/vendor/k8s.io/cluster-bootstrap/token/util/helpers_test.go b/vendor/k8s.io/cluster-bootstrap/token/util/helpers_test.go new file mode 100644 index 0000000000..a1fe6092ff --- /dev/null +++ b/vendor/k8s.io/cluster-bootstrap/token/util/helpers_test.go @@ -0,0 +1,213 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "strings" + "testing" +) + +func TestGenerateBootstrapToken(t *testing.T) { + token, err := GenerateBootstrapToken() + if err != nil { + t.Fatalf("GenerateBootstrapToken returned an unexpected error: %+v", err) + } + if !IsValidBootstrapToken(token) { + t.Errorf("GenerateBootstrapToken didn't generate a valid token: %q", token) + } +} + +func TestRandBytes(t *testing.T) { + var randTest = []int{ + 0, + 1, + 2, + 3, + 100, + } + + for _, rt := range randTest { + actual, err := randBytes(rt) + if err != nil { + t.Errorf("failed randBytes: %v", err) + } + if len(actual) != rt { + t.Errorf("failed randBytes:\n\texpected: %d\n\t actual: %d\n", rt, len(actual)) + } + } +} + +func TestTokenFromIDAndSecret(t *testing.T) { + var tests = []struct { + id string + secret string + expected string + }{ + {"foo", "bar", "foo.bar"}, // should use default + {"abcdef", "abcdef0123456789", "abcdef.abcdef0123456789"}, + {"h", "b", "h.b"}, + } + for _, rt := range tests { + actual := TokenFromIDAndSecret(rt.id, rt.secret) + if actual != rt.expected { + t.Errorf( + "failed TokenFromIDAndSecret:\n\texpected: %s\n\t actual: %s", + rt.expected, + actual, + ) + } + } +} + +func TestIsValidBootstrapToken(t *testing.T) { + var tests = []struct { + token string + expected bool + }{ + {token: "", expected: false}, + {token: ".", expected: false}, + {token: "1234567890123456789012", expected: false}, // invalid parcel size + {token: "12345.1234567890123456", expected: false}, // invalid parcel size + {token: ".1234567890123456", expected: false}, // invalid parcel size + {token: "123456.", expected: false}, // invalid parcel size + {token: "123456:1234567890.123456", expected: false}, // invalid separation + {token: "abcdef:1234567890123456", expected: false}, // invalid separation + {token: "Abcdef.1234567890123456", expected: false}, // invalid token id + {token: "123456.AABBCCDDEEFFGGHH", expected: false}, // invalid token secret + {token: "123456.AABBCCD-EEFFGGHH", expected: false}, // invalid character + {token: "abc*ef.1234567890123456", expected: false}, // invalid character + {token: "abcdef.1234567890123456", expected: true}, + {token: "123456.aabbccddeeffgghh", expected: true}, + {token: "ABCDEF.abcdef0123456789", expected: false}, + {token: "abcdef.abcdef0123456789", expected: true}, + {token: "123456.1234560123456789", expected: true}, + } + for _, rt := range tests { + actual := IsValidBootstrapToken(rt.token) + if actual != rt.expected { + t.Errorf( + "failed IsValidBootstrapToken for the token %q\n\texpected: %t\n\t actual: %t", + rt.token, + rt.expected, + actual, + ) + } + } +} + +func TestIsValidBootstrapTokenID(t *testing.T) { + var tests = []struct { + tokenID string + expected bool + }{ + {tokenID: "", expected: false}, + {tokenID: "1234567890123456789012", expected: false}, + {tokenID: "12345", expected: false}, + {tokenID: "Abcdef", expected: false}, + {tokenID: "ABCDEF", expected: false}, + {tokenID: "abcdef.", expected: false}, + {tokenID: "abcdef", expected: true}, + {tokenID: "123456", expected: true}, + } + for _, rt := range tests { + actual := IsValidBootstrapTokenID(rt.tokenID) + if actual != rt.expected { + t.Errorf( + "failed IsValidBootstrapTokenID for the token %q\n\texpected: %t\n\t actual: %t", + rt.tokenID, + rt.expected, + actual, + ) + } + } +} + +func TestBootstrapTokenSecretName(t *testing.T) { + var tests = []struct { + tokenID string + expected string + }{ + {"foo", "bootstrap-token-foo"}, + {"bar", "bootstrap-token-bar"}, + {"", "bootstrap-token-"}, + {"abcdef", "bootstrap-token-abcdef"}, + } + for _, rt := range tests { + actual := BootstrapTokenSecretName(rt.tokenID) + if actual != rt.expected { + t.Errorf( + "failed BootstrapTokenSecretName:\n\texpected: %s\n\t actual: %s", + rt.expected, + actual, + ) + } + } +} + +func TestValidateBootstrapGroupName(t *testing.T) { + tests := []struct { + name string + input string + valid bool + }{ + {"valid", "system:bootstrappers:foo", true}, + {"valid nested", "system:bootstrappers:foo:bar:baz", true}, + {"valid with dashes and number", "system:bootstrappers:foo-bar-42", true}, + {"invalid uppercase", "system:bootstrappers:Foo", false}, + {"missing prefix", "foo", false}, + {"prefix with no body", "system:bootstrappers:", false}, + {"invalid spaces", "system:bootstrappers: ", false}, + {"invalid asterisk", "system:bootstrappers:*", false}, + {"trailing colon", "system:bootstrappers:foo:", false}, + {"trailing dash", "system:bootstrappers:foo-", false}, + {"script tags", "system:bootstrappers:", false}, + {"too long", "system:bootstrappers:" + strings.Repeat("x", 300), false}, + } + for _, test := range tests { + err := ValidateBootstrapGroupName(test.input) + if err != nil && test.valid { + t.Errorf("test %q: ValidateBootstrapGroupName(%q) returned unexpected error: %v", test.name, test.input, err) + } + if err == nil && !test.valid { + t.Errorf("test %q: ValidateBootstrapGroupName(%q) was supposed to return an error but didn't", test.name, test.input) + } + } +} + +func TestValidateUsages(t *testing.T) { + tests := []struct { + name string + input []string + valid bool + }{ + {"valid of signing", []string{"signing"}, true}, + {"valid of authentication", []string{"authentication"}, true}, + {"all valid", []string{"authentication", "signing"}, true}, + {"single invalid", []string{"authentication", "foo"}, false}, + {"all invalid", []string{"foo", "bar"}, false}, + } + + for _, test := range tests { + err := ValidateUsages(test.input) + if err != nil && test.valid { + t.Errorf("test %q: ValidateUsages(%v) returned unexpected error: %v", test.name, test.input, err) + } + if err == nil && !test.valid { + t.Errorf("test %q: ValidateUsages(%v) was supposed to return an error but didn't", test.name, test.input) + } + } +}