diff --git a/Gopkg.lock b/Gopkg.lock index c1fb6b070a..1874c89f69 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -25,6 +25,45 @@ packages = ["."] revision = "de5bf2ad457846296e2031421a34e2568e304e35" +[[projects]] + name = "github.com/aws/aws-sdk-go" + packages = [ + "aws", + "aws/awserr", + "aws/awsutil", + "aws/client", + "aws/client/metadata", + "aws/corehandlers", + "aws/credentials", + "aws/credentials/ec2rolecreds", + "aws/credentials/endpointcreds", + "aws/credentials/stscreds", + "aws/csm", + "aws/defaults", + "aws/ec2metadata", + "aws/endpoints", + "aws/request", + "aws/session", + "aws/signer/v4", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", + "internal/shareddefaults", + "private/protocol", + "private/protocol/ec2query", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/xml/xmlutil", + "service/ec2", + "service/ec2/ec2iface", + "service/elb", + "service/elb/elbiface", + "service/sts" + ] + revision = "f70339bb6af843c8ab1974381b3f4fcaee2b1a41" + version = "v1.15.5" + [[projects]] branch = "master" name = "github.com/beorn7/perks" @@ -101,6 +140,12 @@ revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" +[[projects]] + name = "github.com/go-ini/ini" + packages = ["."] + revision = "358ee7663966325963d4e8b2e1fbd570c5195153" + version = "v1.38.1" + [[projects]] branch = "master" name = "github.com/go-openapi/jsonpointer" @@ -212,6 +257,11 @@ revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" version = "v0.3.5" +[[projects]] + name = "github.com/jmespath/go-jmespath" + packages = ["."] + revision = "0b12d6b5" + [[projects]] name = "github.com/json-iterator/go" packages = ["."] @@ -318,6 +368,12 @@ ] revision = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a" +[[projects]] + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" + [[projects]] name = "github.com/spf13/pflag" packages = ["."] @@ -632,6 +688,7 @@ name = "k8s.io/client-go" packages = [ "discovery", + "discovery/fake", "informers", "informers/admissionregistration", "informers/admissionregistration/v1alpha1", @@ -673,35 +730,64 @@ "informers/storage/v1alpha1", "informers/storage/v1beta1", "kubernetes", + "kubernetes/fake", "kubernetes/scheme", "kubernetes/typed/admissionregistration/v1alpha1", + "kubernetes/typed/admissionregistration/v1alpha1/fake", "kubernetes/typed/admissionregistration/v1beta1", + "kubernetes/typed/admissionregistration/v1beta1/fake", "kubernetes/typed/apps/v1", + "kubernetes/typed/apps/v1/fake", "kubernetes/typed/apps/v1beta1", + "kubernetes/typed/apps/v1beta1/fake", "kubernetes/typed/apps/v1beta2", + "kubernetes/typed/apps/v1beta2/fake", "kubernetes/typed/authentication/v1", + "kubernetes/typed/authentication/v1/fake", "kubernetes/typed/authentication/v1beta1", + "kubernetes/typed/authentication/v1beta1/fake", "kubernetes/typed/authorization/v1", + "kubernetes/typed/authorization/v1/fake", "kubernetes/typed/authorization/v1beta1", + "kubernetes/typed/authorization/v1beta1/fake", "kubernetes/typed/autoscaling/v1", + "kubernetes/typed/autoscaling/v1/fake", "kubernetes/typed/autoscaling/v2beta1", + "kubernetes/typed/autoscaling/v2beta1/fake", "kubernetes/typed/batch/v1", + "kubernetes/typed/batch/v1/fake", "kubernetes/typed/batch/v1beta1", + "kubernetes/typed/batch/v1beta1/fake", "kubernetes/typed/batch/v2alpha1", + "kubernetes/typed/batch/v2alpha1/fake", "kubernetes/typed/certificates/v1beta1", + "kubernetes/typed/certificates/v1beta1/fake", "kubernetes/typed/core/v1", + "kubernetes/typed/core/v1/fake", "kubernetes/typed/events/v1beta1", + "kubernetes/typed/events/v1beta1/fake", "kubernetes/typed/extensions/v1beta1", + "kubernetes/typed/extensions/v1beta1/fake", "kubernetes/typed/networking/v1", + "kubernetes/typed/networking/v1/fake", "kubernetes/typed/policy/v1beta1", + "kubernetes/typed/policy/v1beta1/fake", "kubernetes/typed/rbac/v1", + "kubernetes/typed/rbac/v1/fake", "kubernetes/typed/rbac/v1alpha1", + "kubernetes/typed/rbac/v1alpha1/fake", "kubernetes/typed/rbac/v1beta1", + "kubernetes/typed/rbac/v1beta1/fake", "kubernetes/typed/scheduling/v1alpha1", + "kubernetes/typed/scheduling/v1alpha1/fake", "kubernetes/typed/settings/v1alpha1", + "kubernetes/typed/settings/v1alpha1/fake", "kubernetes/typed/storage/v1", + "kubernetes/typed/storage/v1/fake", "kubernetes/typed/storage/v1alpha1", + "kubernetes/typed/storage/v1alpha1/fake", "kubernetes/typed/storage/v1beta1", + "kubernetes/typed/storage/v1beta1/fake", "listers/admissionregistration/v1alpha1", "listers/admissionregistration/v1beta1", "listers/apps/v1", @@ -729,6 +815,7 @@ "pkg/version", "rest", "rest/watch", + "testing", "tools/auth", "tools/cache", "tools/clientcmd", @@ -791,8 +878,10 @@ "pkg/apis/cluster/common", "pkg/apis/cluster/v1alpha1", "pkg/client/clientset_generated/clientset", + "pkg/client/clientset_generated/clientset/fake", "pkg/client/clientset_generated/clientset/scheme", "pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1", + "pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1/fake", "pkg/client/informers_generated/externalversions", "pkg/client/informers_generated/externalversions/cluster", "pkg/client/informers_generated/externalversions/cluster/v1alpha1", @@ -810,6 +899,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cdc10cd8ccd232cdec9f8e7e969f8734a337dbcdadab8d271e460e88879a08da" + inputs-digest = "faa7adf72b05e8df2a49914cf4e1e2df86fdec4aa5014683e78eee5ee2783dc1" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 08acff8c95..5f252fdbba 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -35,3 +35,7 @@ required = [ [[constraint]] name = "k8s.io/code-generator" branch = "release-1.9" + +[[constraint]] + name = "github.com/aws/aws-sdk-go" + version = "v1.15.5" diff --git a/Makefile b/Makefile index 5d253e2d03..327fd516b5 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,11 @@ check: depend fmt vet test: go test -race -cover ./cmd/... ./cloud/... +integration: + go test -v sigs.k8s.io/cluster-api-provider-aws/test/integration + fmt: hack/verify-gofmt.sh vet: - go vet ./... \ No newline at end of file + go vet ./... diff --git a/cloud/aws/actuators/machine/actuator.go b/cloud/aws/actuators/machine/actuator.go index 42d13d0f4a..fb58fb0d85 100644 --- a/cloud/aws/actuators/machine/actuator.go +++ b/cloud/aws/actuators/machine/actuator.go @@ -1,29 +1,56 @@ -// Copyright © 2018 The Kubernetes Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package machine import ( + "encoding/base64" "fmt" "github.com/golang/glog" + log "github.com/sirupsen/logrus" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/providerconfig/v1alpha1" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + clusterclient "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset" client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + + awsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" + clustoplog "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/logging" +) + +const ( + userDataSecretKey = "userDataKey" + ec2InstanceIDNotFoundCode = "InvalidInstanceID.NotFound" ) -// Actuator is responsible for performing machine reconciliation +// Actuator is the AWS-specific actuator for the Cluster API machine controller type Actuator struct { - clusterClient client.ClusterInterface + kubeClient kubernetes.Interface + clusterClient clusterclient.Interface + logger *log.Entry + awsClientBuilder awsclient.AwsClientBuilderFuncType } // ActuatorParams holds parameter information for Actuator @@ -31,33 +58,384 @@ type ActuatorParams struct { ClusterClient client.ClusterInterface } -// NewActuator creates a new Actuator -func NewActuator(params ActuatorParams) (*Actuator, error) { - return &Actuator{ - clusterClient: params.ClusterClient, - }, nil +// NewActuator returns a new AWS Actuator +func NewActuator(kubeClient kubernetes.Interface, clusterClient clusterclient.Interface, logger *log.Entry, awsClientBuilder awsclient.AwsClientBuilderFuncType) (*Actuator, error) { + actuator := &Actuator{ + kubeClient: kubeClient, + clusterClient: clusterClient, + logger: logger, + awsClientBuilder: awsClientBuilder, + } + return actuator, nil } -// Create creates a machine and is invoked by the Machine Controller +// Create runs a new EC2 instance func (a *Actuator) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { - glog.Infof("Creating machine %v for cluster %v.", machine.Name, cluster.Name) - return fmt.Errorf("TODO: Not yet implemented") + mLog := clustoplog.WithMachine(a.logger, machine) + mLog.Info("creating machine") + instance, err := a.CreateMachine(cluster, machine) + if err != nil { + mLog.Errorf("error creating machine: %v", err) + return err + } + + // TODO(csrwng): + // Part of the status that gets updated when the machine gets created is the PublicIP. + // However, after a call to runInstance, most of the time a PublicIP is not yet allocated. + // If we don't yet have complete status (ie. the instance state is Pending, instead of Running), + // maybe we should return an error so the machine controller keeps retrying until we have complete status we can set. + return a.updateStatus(machine, instance, mLog) +} + +// removeStoppedMachine removes all instances of a specific machine that are in a stopped state. +func (a *Actuator) removeStoppedMachine(machine *clusterv1.Machine, client awsclient.Client, mLog log.FieldLogger) error { + instances, err := GetStoppedInstances(machine, client) + if err != nil { + return fmt.Errorf("Error getting stopped instances: %v", err) + } + + if len(instances) == 0 { + mLog.Infof("no stopped instances found for machine %v", machine.Name) + return nil + } + + return TerminateInstances(client, instances, mLog) +} + +// CreateMachine starts a new AWS instance as described by the cluster and machine resources +func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (*ec2.Instance, error) { + mLog := clustoplog.WithMachine(a.logger, machine) + + machineProviderConfig, err := MachineProviderConfigFromClusterAPIMachineSpec(&machine.Spec) + if err != nil { + return nil, err + } + + client, err := a.awsClientBuilder(a.kubeClient, machineProviderConfig.CredentialsSecret.Name, machine.Namespace, machineProviderConfig.Placement.Region) + if err != nil { + return nil, fmt.Errorf("unable to obtain AWS client: %v", err) + } + + // We explicitly do NOT want to remove stopped masters. + if !MachineIsMaster(machine) { + // Prevent having a lot of stopped nodes sitting around. + err = a.removeStoppedMachine(machine, client, mLog) + if err != nil { + return nil, fmt.Errorf("unable to remove stopped nodes: %v", err) + } + } + + // TODO(jchaloup): resolve ARN and/or Filters as well + if machineProviderConfig.AMI.ID == nil { + return nil, fmt.Errorf("machine does not have an AWS image set") + } + + // Get AMI to use + amiName := *machineProviderConfig.AMI.ID + + mLog.Debugf("Describing AMI %s", amiName) + imageIds := []*string{aws.String(amiName)} + describeImagesRequest := ec2.DescribeImagesInput{ + ImageIds: imageIds, + } + describeAMIResult, err := client.DescribeImages(&describeImagesRequest) + if err != nil { + return nil, fmt.Errorf("error describing AMI %s: %v", amiName, err) + } + if len(describeAMIResult.Images) != 1 { + return nil, fmt.Errorf("Unexpected number of images returned: %d", len(describeAMIResult.Images)) + } + + var securityGroupIds []*string + for _, g := range machineProviderConfig.SecurityGroups { + groupID := *g.ID + securityGroupIds = append(securityGroupIds, &groupID) + } + + // build list of networkInterfaces (just 1 for now) + var networkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ + { + DeviceIndex: aws.Int64(machineProviderConfig.DeviceIndex), + AssociatePublicIpAddress: machineProviderConfig.PublicIP, + SubnetId: machineProviderConfig.Subnet.ID, + Groups: securityGroupIds, + }, + } + + // Add tags to the created machine + tagList := []*ec2.Tag{} + for _, tag := range machineProviderConfig.Tags { + tagList = append(tagList, &ec2.Tag{Key: aws.String(tag.Name), Value: aws.String(tag.Value)}) + } + tagList = append(tagList, []*ec2.Tag{ + {Key: aws.String("clusterid"), Value: aws.String(cluster.Name)}, + {Key: aws.String("kubernetes.io/cluster/" + cluster.Name), Value: aws.String(cluster.Name)}, + {Key: aws.String("Name"), Value: aws.String(machine.Name)}, + }...) + + tagInstance := &ec2.TagSpecification{ + ResourceType: aws.String("instance"), + Tags: tagList, + } + tagVolume := &ec2.TagSpecification{ + ResourceType: aws.String("volume"), + Tags: []*ec2.Tag{{Key: aws.String("clusterid"), Value: aws.String(cluster.Name)}}, + } + + userData := []byte{} + if machineProviderConfig.UserDataSecret != nil { + userDataSecret, err := a.kubeClient.CoreV1().Secrets(machine.Namespace).Get(machineProviderConfig.UserDataSecret.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if data, exists := userDataSecret.Data[userDataSecretKey]; exists { + userData = data + } else { + glog.Warning("Secret %v/%v does not have %q field set. Thus, no user data applied when creating an instance.", machine.Namespace, machineProviderConfig.UserDataSecret.Name, userDataSecretKey) + } + } + + userDataEnc := base64.StdEncoding.EncodeToString(userData) + + inputConfig := ec2.RunInstancesInput{ + ImageId: describeAMIResult.Images[0].ImageId, + InstanceType: aws.String(machineProviderConfig.InstanceType), + // Only a single instance of the AWS instance allowed + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + KeyName: machineProviderConfig.KeyName, + IamInstanceProfile: &ec2.IamInstanceProfileSpecification{ + Name: aws.String(*machineProviderConfig.IAMInstanceProfile.ID), + }, + TagSpecifications: []*ec2.TagSpecification{tagInstance, tagVolume}, + NetworkInterfaces: networkInterfaces, + UserData: &userDataEnc, + } + + runResult, err := client.RunInstances(&inputConfig) + if err != nil { + return nil, fmt.Errorf("cannot create EC2 instance: %v", err) + } + + if runResult == nil || len(runResult.Instances) != 1 { + mLog.Errorf("unexpected reservation creating instances: %v", runResult) + return nil, fmt.Errorf("unexpected reservation creating instance") + } + + // TOOD(csrwng): + // One issue we have right now with how this works, is that usually at the end of the RunInstances call, + // the instance state is not yet 'Running'. Rather it is 'Pending'. Therefore the status + // is not yet complete (like PublicIP). One possible fix would be to wait and poll here + // until the instance is in the Running state. The other approach is to return an error + // so that the machine is requeued and in the exists function return false if the status doesn't match. + // That would require making the create re-entrant so we can just update the status. + return runResult.Instances[0], nil } -// Delete deletes a machine and is invoked by the Machine Controller +// Delete deletes a machine and updates its finalizer func (a *Actuator) Delete(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { - glog.Infof("Deleting machine %v for cluster %v.", machine.Name, cluster.Name) - return fmt.Errorf("TODO: Not yet implemented") + mLog := clustoplog.WithMachine(a.logger, machine) + mLog.Info("deleting machine") + if err := a.DeleteMachine(cluster, machine); err != nil { + mLog.Errorf("error deleting machine: %v", err) + return err + } + return nil +} + +// DeleteMachine deletes an AWS instance +func (a *Actuator) DeleteMachine(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { + mLog := clustoplog.WithMachine(a.logger, machine) + + machineProviderConfig, err := MachineProviderConfigFromClusterAPIMachineSpec(&machine.Spec) + if err != nil { + return err + } + + region := machineProviderConfig.Placement.Region + client, err := a.awsClientBuilder(a.kubeClient, machineProviderConfig.CredentialsSecret.Name, machine.Namespace, region) + if err != nil { + return fmt.Errorf("error getting EC2 client: %v", err) + } + + instances, err := GetRunningInstances(machine, client) + if err != nil { + return err + } + if len(instances) == 0 { + mLog.Warnf("no instances found to delete for machine") + return nil + } + + return TerminateInstances(client, instances, mLog) } -// Update updates a machine and is invoked by the Machine Controller +// Update attempts to sync machine state with an existing instance. Today this just updates status +// for details that may have changed. (IPs and hostnames) We do not currently support making any +// changes to actual machines in AWS. Instead these will be replaced via MachineDeployments. func (a *Actuator) Update(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error { - glog.Infof("Updating machine %v for cluster %v.", machine.Name, cluster.Name) - return fmt.Errorf("TODO: Not yet implemented") + mLog := clustoplog.WithMachine(a.logger, machine) + mLog.Debugf("updating machine") + + machineProviderConfig, err := MachineProviderConfigFromClusterAPIMachineSpec(&machine.Spec) + if err != nil { + return err + } + + region := machineProviderConfig.Placement.Region + mLog.WithField("region", region).Debugf("obtaining EC2 client for region") + client, err := a.awsClientBuilder(a.kubeClient, machineProviderConfig.CredentialsSecret.Name, machine.Namespace, region) + if err != nil { + return fmt.Errorf("unable to obtain EC2 client: %v", err) + } + + instances, err := GetRunningInstances(machine, client) + mLog.Debugf("found %d instances for machine", len(instances)) + if err != nil { + return err + } + + // Parent controller should prevent this from ever happening by calling Exists and then Create, + // but instance could be deleted between the two calls. + if len(instances) == 0 { + mLog.Warnf("attempted to update machine but no instances found") + // Update status to clear out machine details. + err := a.updateStatus(machine, nil, mLog) + if err != nil { + return err + } + return fmt.Errorf("attempted to update machine but no instances found") + } + newestInstance, terminateInstances := SortInstances(instances) + + // In very unusual circumstances, there could be more than one machine running matching this + // machine name and cluster ID. In this scenario we will keep the newest, and delete all others. + mLog = mLog.WithField("instanceID", *newestInstance.InstanceId) + mLog.Debug("instance found") + + if len(instances) > 1 { + err = TerminateInstances(client, terminateInstances, mLog) + if err != nil { + return err + } + + } + + // We do not support making changes to pre-existing instances, just update status. + return a.updateStatus(machine, newestInstance, mLog) } -// Exists test for the existance of a machine and is invoked by the Machine Controller +// Exists determines if the given machine currently exists. For AWS we query for instances in +// running state, with a matching name tag, to determine a match. func (a *Actuator) Exists(cluster *clusterv1.Cluster, machine *clusterv1.Machine) (bool, error) { - glog.Info("Checking if machine %v for cluster %v exists.", machine.Name, cluster.Name) - return false, fmt.Errorf("TODO: Not yet implemented") + mLog := clustoplog.WithMachine(a.logger, machine) + mLog.Debugf("checking if machine exists") + + machineProviderConfig, err := MachineProviderConfigFromClusterAPIMachineSpec(&machine.Spec) + if err != nil { + return false, err + } + + region := machineProviderConfig.Placement.Region + client, err := a.awsClientBuilder(a.kubeClient, machineProviderConfig.CredentialsSecret.Name, machine.Namespace, region) + if err != nil { + return false, fmt.Errorf("error getting EC2 client: %v", err) + } + + instances, err := GetRunningInstances(machine, client) + if err != nil { + return false, err + } + if len(instances) == 0 { + mLog.Debug("instance does not exist") + return false, nil + } + + // If more than one result was returned, it will be handled in Update. + mLog.Debug("instance exists") + return true, nil +} + +// updateStatus calculates the new machine status, checks if anything has changed, and updates if so. +func (a *Actuator) updateStatus(machine *clusterv1.Machine, instance *ec2.Instance, mLog log.FieldLogger) error { + + mLog.Debug("updating status") + + // Starting with a fresh status as we assume full control of it here. + awsStatus, err := AWSMachineProviderStatusFromClusterAPIMachine(machine) + if err != nil { + return err + } + // Save this, we need to check if it changed later. + networkAddresses := []corev1.NodeAddress{} + + // Instance may have existed but been deleted outside our control, clear it's status if so: + if instance == nil { + awsStatus.InstanceID = nil + awsStatus.InstanceState = nil + } else { + awsStatus.InstanceID = instance.InstanceId + awsStatus.InstanceState = instance.State.Name + if instance.PublicIpAddress != nil { + networkAddresses = append(networkAddresses, corev1.NodeAddress{ + Type: corev1.NodeExternalIP, + Address: *instance.PublicIpAddress, + }) + } + if instance.PrivateIpAddress != nil { + networkAddresses = append(networkAddresses, corev1.NodeAddress{ + Type: corev1.NodeInternalIP, + Address: *instance.PrivateIpAddress, + }) + } + if instance.PublicDnsName != nil { + networkAddresses = append(networkAddresses, corev1.NodeAddress{ + Type: corev1.NodeExternalDNS, + Address: *instance.PublicDnsName, + }) + } + if instance.PrivateDnsName != nil { + networkAddresses = append(networkAddresses, corev1.NodeAddress{ + Type: corev1.NodeInternalDNS, + Address: *instance.PrivateDnsName, + }) + } + } + mLog.Debug("finished calculating AWS status") + + // TODO(jchaloup): do we really need to update tis? + // origInstanceID := awsStatus.InstanceID + // if !StringPtrsEqual(origInstanceID, awsStatus.InstanceID) { + // mLog.Debug("AWS instance ID changed, clearing LastELBSync to trigger adding to ELBs") + // awsStatus.LastELBSync = nil + // } + + awsStatusRaw, err := EncodeAWSMachineProviderStatus(awsStatus) + if err != nil { + mLog.Errorf("error encoding AWS provider status: %v", err) + return err + } + + machineCopy := machine.DeepCopy() + machineCopy.Status.ProviderStatus = awsStatusRaw + machineCopy.Status.Addresses = networkAddresses + + if !equality.Semantic.DeepEqual(machine.Status, machineCopy.Status) { + mLog.Info("machine status has changed, updating") + machineCopy.Status.LastUpdated = metav1.Now() + + _, err := a.clusterClient.ClusterV1alpha1().Machines(machineCopy.Namespace).UpdateStatus(machineCopy) + if err != nil { + mLog.Errorf("error updating machine status: %v", err) + return err + } + } else { + mLog.Debug("status unchanged") + } + return nil +} + +func getClusterID(machine *clusterv1.Machine) (string, bool) { + clusterID, ok := machine.Labels[providerconfigv1.ClusterNameLabel] + return clusterID, ok } diff --git a/cloud/aws/actuators/machine/actuator_test.go b/cloud/aws/actuators/machine/actuator_test.go new file mode 100644 index 0000000000..8b9a877ce5 --- /dev/null +++ b/cloud/aws/actuators/machine/actuator_test.go @@ -0,0 +1,196 @@ +package machine + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/fake" + + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + apiv1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/aws-sdk-go/aws" + awsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" + fakeawsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client/fake" + providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/providerconfig/v1alpha1" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +const ( + controllerLogName = "awsMachine" + defaultLogLevel = "info" + + defaultNamespace = "default" + defaultAvailabilityZone = "us-east-1a" + region = "us-east-1" + awsCredentialsSecretName = "aws-credentials-secret" + userDataSecretName = "aws-actuator-user-data-secret" + + keyName = "aws-actuator-key-name" + clusterName = "aws-actuator-cluster" +) + +const userDataBlob = `#cloud-config +write_files: +- path: /root/node_bootstrap/node_settings.yaml + owner: 'root:root' + permissions: '0640' + content: | + node_config_name: node-config-master +runcmd: +- [ cat, /root/node_bootstrap/node_settings.yaml] +` + +func testMachineAPIResources(clusterID string) (*clusterv1.Machine, *clusterv1.Cluster, *apiv1.Secret, *apiv1.Secret, error) { + awsCredentialsSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: awsCredentialsSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + awsclient.AwsCredsSecretIDKey: []byte(os.Getenv("AWS_ACCESS_KEY_ID")), + awsclient.AwsCredsSecretAccessKey: []byte(os.Getenv("AWS_SECRET_ACCESS_KEY")), + }, + } + + userDataSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + userDataSecretKey: []byte(userDataBlob), + }, + } + + machinePc := &providerconfigv1.AWSMachineProviderConfig{ + AMI: providerconfigv1.AWSResourceReference{ + ID: aws.String("ami-a9acbbd6"), + }, + CredentialsSecret: &corev1.LocalObjectReference{ + Name: awsCredentialsSecretName, + }, + InstanceType: "m4.xlarge", + Placement: providerconfigv1.Placement{ + Region: region, + AvailabilityZone: defaultAvailabilityZone, + }, + Subnet: providerconfigv1.AWSResourceReference{ + ID: aws.String("subnet-0e56b13a64ff8a941"), + }, + IAMInstanceProfile: &providerconfigv1.AWSResourceReference{ + ID: aws.String("openshift_master_launch_instances"), + }, + KeyName: aws.String(keyName), + UserDataSecret: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + Tags: []providerconfigv1.TagSpecification{ + {Name: "openshift-node-group-config", Value: "node-config-master"}, + {Name: "host-type", Value: "master"}, + {Name: "sub-host-type", Value: "default"}, + }, + SecurityGroups: []providerconfigv1.AWSResourceReference{ + {ID: aws.String("sg-00868b02fbe29de17")}, // aws-actuator + {ID: aws.String("sg-0a4658991dc5eb40a")}, // aws-actuator_master + {ID: aws.String("sg-009a70e28fa4ba84e")}, // aws-actuator_master_k8s + {ID: aws.String("sg-07323d56fb932c84c")}, // aws-actuator_infra + {ID: aws.String("sg-08b1ffd32874d59a2")}, // aws-actuator_infra_k8s + }, + PublicIP: aws.Bool(true), + } + + var buf bytes.Buffer + if err := providerconfigv1.Encoder.Encode(machinePc, &buf); err != nil { + return nil, nil, nil, nil, fmt.Errorf("encoding failed: %v", err) + } + + machine := &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aws-actuator-testing-machine", + Namespace: defaultNamespace, + Labels: map[string]string{ + providerconfigv1.ClusterNameLabel: clusterID, + providerconfigv1.MachineRoleLabel: "infra", + providerconfigv1.MachineTypeLabel: "master", + }, + }, + + Spec: clusterv1.MachineSpec{ + ProviderConfig: clusterv1.ProviderConfig{ + Value: &runtime.RawExtension{Raw: buf.Bytes()}, + }, + }, + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterID, + Namespace: defaultNamespace, + }, + } + + return machine, cluster, awsCredentialsSecret, userDataSecret, nil +} + +func TestCreateAndDeleteMachine(t *testing.T) { + + // kube client is needed to fetch aws credentials: + // - kubeClient.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + // cluster client for updating machine statues + // - clusterClient.ClusterV1alpha1().Machines(machineCopy.Namespace).UpdateStatus(machineCopy) + + machine, cluster, awsCredentialsSecret, userDataSecret, err := testMachineAPIResources(clusterName) + if err != nil { + t.Fatal(err) + } + + fakeKubeClient := kubernetesfake.NewSimpleClientset(awsCredentialsSecret, userDataSecret) + fakeClient := fake.NewSimpleClientset(machine) + logger := log.WithField("controller", controllerLogName) + + actuator, err := NewActuator(fakeKubeClient, fakeClient, logger, fakeawsclient.NewClient) + if err != nil { + t.Fatalf("Could not create Openstack machine actuator: %v", err) + } + + // Create a machine + if err := actuator.Create(cluster, machine); err != nil { + t.Errorf("Unable to create instance for machine: %v", err) + } + + // Get the machine + if exists, err := actuator.Exists(cluster, machine); err != nil || !exists { + t.Errorf("Instance for %v does not exists: %v", strings.Join([]string{machine.Namespace, machine.Name}, "/"), err) + } else { + t.Logf("Instance for %v exists", strings.Join([]string{machine.Namespace, machine.Name}, "/")) + } + + // TODO(jchaloup): Wait until the machine is ready + + // Update a machine + if err := actuator.Update(cluster, machine); err != nil { + t.Errorf("Unable to create instance for machine: %v", err) + } + + // Get the machine + if exists, err := actuator.Exists(cluster, machine); err != nil || !exists { + t.Errorf("Instance for %v does not exists: %v", strings.Join([]string{machine.Namespace, machine.Name}, "/"), err) + } else { + t.Logf("Instance for %v exists", strings.Join([]string{machine.Namespace, machine.Name}, "/")) + } + + // Delete a machine + if err := actuator.Delete(cluster, machine); err != nil { + t.Errorf("Unable to delete instance for machine: %v", err) + } +} diff --git a/cloud/aws/actuators/machine/utils.go b/cloud/aws/actuators/machine/utils.go new file mode 100644 index 0000000000..2367bfb6f1 --- /dev/null +++ b/cloud/aws/actuators/machine/utils.go @@ -0,0 +1,266 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machine + +import ( + "bytes" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + jsonserializer "k8s.io/apimachinery/pkg/runtime/serializer/json" + + awsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" + providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/providerconfig/v1alpha1" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// SortInstances will examine the given slice of instances and return the current active instance for +// the machine, as well as a slice of all other instances which the caller may want to terminate. The +// active instance is calculated as the most recently launched instance. +// This function should only be called with running instances, not those which are stopped or +// terminated. +func SortInstances(instances []*ec2.Instance) (*ec2.Instance, []*ec2.Instance) { + if len(instances) == 0 { + return nil, []*ec2.Instance{} + } + var newestInstance *ec2.Instance + inactiveInstances := make([]*ec2.Instance, 0, len(instances)-1) + for _, i := range instances { + if newestInstance == nil { + newestInstance = i + continue + } + tempInstance := chooseNewest(newestInstance, i) + if *tempInstance.InstanceId != *newestInstance.InstanceId { + inactiveInstances = append(inactiveInstances, newestInstance) + } else { + inactiveInstances = append(inactiveInstances, i) + } + newestInstance = tempInstance + } + return newestInstance, inactiveInstances +} + +func chooseNewest(instance1, instance2 *ec2.Instance) *ec2.Instance { + if instance1.LaunchTime == nil && instance2.LaunchTime == nil { + // No idea what to do here, should not be possible, just return the first. + return instance1 + } + if instance1.LaunchTime != nil && instance2.LaunchTime == nil { + return instance1 + } + if instance1.LaunchTime == nil && instance2.LaunchTime != nil { + return instance2 + } + if (*instance1.LaunchTime).After(*instance2.LaunchTime) { + return instance1 + } + return instance2 +} + +// GetInstance returns the AWS instance for a given machine. If multiple instances match our machine, +// the most recently launched will be returned. If no instance exists, an error will be returned. +func GetInstance(machine *clusterv1.Machine, client awsclient.Client) (*ec2.Instance, error) { + instances, err := GetRunningInstances(machine, client) + if err != nil { + return nil, err + } + if len(instances) == 0 { + return nil, fmt.Errorf("no instance found for machine: %s", machine.Name) + } + + instance, _ := SortInstances(instances) + return instance, nil +} + +// GetRunningInstances returns all running instances that have a tag matching our machine name, +// and cluster ID. +func GetRunningInstances(machine *clusterv1.Machine, client awsclient.Client) ([]*ec2.Instance, error) { + runningInstanceStateFilter := []*string{aws.String(ec2.InstanceStateNameRunning), aws.String(ec2.InstanceStateNamePending)} + return GetInstances(machine, client, runningInstanceStateFilter) +} + +// GetStoppedInstances returns all stopped instances that have a tag matching our machine name, +// and cluster ID. +func GetStoppedInstances(machine *clusterv1.Machine, client awsclient.Client) ([]*ec2.Instance, error) { + stoppedInstanceStateFilter := []*string{aws.String(ec2.InstanceStateNameStopped), aws.String(ec2.InstanceStateNameStopping)} + return GetInstances(machine, client, stoppedInstanceStateFilter) +} + +// GetInstances returns all instances that have a tag matching our machine name, +// and cluster ID. +func GetInstances(machine *clusterv1.Machine, client awsclient.Client, instanceStateFilter []*string) ([]*ec2.Instance, error) { + + machineName := machine.Name + + clusterID, ok := getClusterID(machine) + if !ok { + return []*ec2.Instance{}, fmt.Errorf("unable to get cluster ID for machine: %q", machine.Name) + } + + requestFilters := []*ec2.Filter{ + { + Name: aws.String("tag:Name"), + Values: []*string{&machineName}, + }, + { + Name: aws.String("tag:clusterid"), + Values: []*string{&clusterID}, + }, + } + + if instanceStateFilter != nil { + requestFilters = append(requestFilters, &ec2.Filter{ + Name: aws.String("instance-state-name"), + Values: instanceStateFilter, + }) + } + + // Query instances with our machine's name, and in running/pending state. + request := &ec2.DescribeInstancesInput{ + Filters: requestFilters, + } + + result, err := client.DescribeInstances(request) + if err != nil { + return []*ec2.Instance{}, err + } + + instances := make([]*ec2.Instance, 0, len(result.Reservations)) + for _, reservation := range result.Reservations { + for _, instance := range reservation.Instances { + instances = append(instances, instance) + } + } + + return instances, nil +} + +// TerminateInstances terminates all provided instances with a single EC2 request. +func TerminateInstances(client awsclient.Client, instances []*ec2.Instance, mLog log.FieldLogger) error { + instanceIDs := []*string{} + // Cleanup all older instances: + for _, instance := range instances { + mLog.WithFields(log.Fields{ + "instanceID": *instance.InstanceId, + "state": *instance.State.Name, + "launchTime": *instance.LaunchTime, + }).Warn("cleaning up extraneous instance for machine") + instanceIDs = append(instanceIDs, instance.InstanceId) + } + for _, instanceID := range instanceIDs { + mLog.WithField("instanceID", *instanceID).Info("terminating instance") + } + + terminateInstancesRequest := &ec2.TerminateInstancesInput{ + InstanceIds: instanceIDs, + } + _, err := client.TerminateInstances(terminateInstancesRequest) + if err != nil { + mLog.Errorf("error terminating instances: %v", err) + return fmt.Errorf("error terminating instances: %v", err) + } + return nil +} + +// MachineConfigProviderFromClusterAPIMachineSpec gets the machine provider config MachineSetSpec from the +// specified cluster-api MachineSpec. +func MachineProviderConfigFromClusterAPIMachineSpec(ms *clusterv1.MachineSpec) (*providerconfigv1.AWSMachineProviderConfig, error) { + if ms.ProviderConfig.Value == nil { + return nil, fmt.Errorf("no Value in ProviderConfig") + } + obj, gvk, err := providerconfigv1.Codecs.UniversalDecoder(providerconfigv1.SchemeGroupVersion).Decode([]byte(ms.ProviderConfig.Value.Raw), nil, nil) + if err != nil { + return nil, err + } + spec, ok := obj.(*providerconfigv1.AWSMachineProviderConfig) + if !ok { + return nil, fmt.Errorf("unexpected object when parsing machine provider config: %#v", gvk) + } + return spec, nil +} + +// AWSMachineProviderStatusFromClusterAPIMachine gets the machine provider status from the specified machine. +func AWSMachineProviderStatusFromClusterAPIMachine(m *clusterv1.Machine) (*providerconfigv1.AWSMachineProviderStatus, error) { + return AWSMachineProviderStatusFromMachineStatus(&m.Status) +} + +// AWSMachineProviderStatusFromMachineStatus gets the machine provider status from the specified machine status. +func AWSMachineProviderStatusFromMachineStatus(s *clusterv1.MachineStatus) (*providerconfigv1.AWSMachineProviderStatus, error) { + if s.ProviderStatus == nil { + return &providerconfigv1.AWSMachineProviderStatus{}, nil + } + obj, gvk, err := providerconfigv1.Codecs.UniversalDecoder(providerconfigv1.SchemeGroupVersion).Decode([]byte(s.ProviderStatus.Raw), nil, nil) + if err != nil { + return nil, err + } + status, ok := obj.(*providerconfigv1.AWSMachineProviderStatus) + if !ok { + return nil, fmt.Errorf("unexpected object: %#v", gvk) + } + return status, nil +} + +// EncodeAWSMachineProviderStatus encodes the machine status into RawExtension +func EncodeAWSMachineProviderStatus(awsStatus *providerconfigv1.AWSMachineProviderStatus) (*runtime.RawExtension, error) { + awsStatus.TypeMeta = metav1.TypeMeta{ + APIVersion: clusterv1.SchemeGroupVersion.String(), + Kind: "AWSMachineProviderStatus", + } + serializer := jsonserializer.NewSerializer(jsonserializer.DefaultMetaFactory, providerconfigv1.Scheme, providerconfigv1.Scheme, false) + var buffer bytes.Buffer + err := serializer.Encode(awsStatus, &buffer) + if err != nil { + return nil, err + } + return &runtime.RawExtension{ + Raw: bytes.TrimSpace(buffer.Bytes()), + }, nil +} + +// MachineIsMaster returns true if the machine is part of a cluster's control plane +func MachineIsMaster(machine *clusterv1.Machine) bool { + if machineType, exists := machine.ObjectMeta.Labels[providerconfigv1.MachineTypeLabel]; exists && machineType == "master" { + return true + } + return false +} + +// MachineIsInfra returns true if the machine is part of a cluster's infra plane +func MachineIsInfra(machine *clusterv1.Machine) bool { + if machineRole, exists := machine.ObjectMeta.Labels[providerconfigv1.MachineRoleLabel]; exists && machineRole == "infra" { + return true + } + return false +} + +// StringPtrsEqual safely returns true if the value for each string pointer is equal, or both are nil. +func StringPtrsEqual(s1, s2 *string) bool { + if s1 == s2 { + return true + } + if s1 == nil || s2 == nil { + return false + } + return *s1 == *s2 +} diff --git a/cloud/aws/client/client.go b/cloud/aws/client/client.go new file mode 100644 index 0000000000..eccfe24056 --- /dev/null +++ b/cloud/aws/client/client.go @@ -0,0 +1,130 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elb/elbiface" +) + +//go:generate mockgen -source=./client.go -destination=./mock/client_generated.go -package=mock + +const ( + AwsCredsSecretIDKey = "awsAccessKeyId" + AwsCredsSecretAccessKey = "awsSecretAccessKey" +) + +type AwsClientBuilderFuncType func(kubeClient kubernetes.Interface, secretName, namespace, region string) (Client, error) + +// Client is a wrapper object for actual AWS SDK clients to allow for easier testing. +type Client interface { + DescribeImages(*ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) + DescribeVpcs(*ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) + DescribeSubnets(*ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) + DescribeSecurityGroups(*ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) + RunInstances(*ec2.RunInstancesInput) (*ec2.Reservation, error) + DescribeInstances(*ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) + TerminateInstances(*ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) + + RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) +} + +type awsClient struct { + ec2Client ec2iface.EC2API + elbClient elbiface.ELBAPI +} + +func (c *awsClient) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { + return c.ec2Client.DescribeImages(input) +} + +func (c *awsClient) DescribeVpcs(input *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) { + return c.ec2Client.DescribeVpcs(input) +} + +func (c *awsClient) DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) { + return c.ec2Client.DescribeSubnets(input) +} + +func (c *awsClient) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) { + return c.ec2Client.DescribeSecurityGroups(input) +} + +func (c *awsClient) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) { + return c.ec2Client.RunInstances(input) +} + +func (c *awsClient) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + return c.ec2Client.DescribeInstances(input) +} + +func (c *awsClient) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { + return c.ec2Client.TerminateInstances(input) +} + +func (c *awsClient) RegisterInstancesWithLoadBalancer(input *elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) { + return c.elbClient.RegisterInstancesWithLoadBalancer(input) +} + +// NewClient creates our client wrapper object for the actual AWS clients we use. +// For authentication the underlying clients will use either the cluster AWS credentials +// secret if defined (i.e. in the root cluster), +// otherwise the IAM profile of the master where the actuator will run. (target clusters) +func NewClient(kubeClient kubernetes.Interface, secretName, namespace, region string) (Client, error) { + awsConfig := &aws.Config{Region: aws.String(region)} + + if secretName != "" { + secret, err := kubeClient.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + accessKeyID, ok := secret.Data[AwsCredsSecretIDKey] + if !ok { + return nil, fmt.Errorf("AWS credentials secret %v did not contain key %v", + secretName, AwsCredsSecretIDKey) + } + secretAccessKey, ok := secret.Data[AwsCredsSecretAccessKey] + if !ok { + return nil, fmt.Errorf("AWS credentials secret %v did not contain key %v", + secretName, AwsCredsSecretAccessKey) + } + + awsConfig.Credentials = credentials.NewStaticCredentials( + string(accessKeyID), string(secretAccessKey), "") + } + + // Otherwise default to relying on the IAM role of the masters where the actuator is running: + s, err := session.NewSession(awsConfig) + if err != nil { + return nil, err + } + + return &awsClient{ + ec2Client: ec2.New(s), + elbClient: elb.New(s), + }, nil +} diff --git a/cloud/aws/client/fake/fake.go b/cloud/aws/client/fake/fake.go new file mode 100644 index 0000000000..be5a71de1a --- /dev/null +++ b/cloud/aws/client/fake/fake.go @@ -0,0 +1,106 @@ +package fake + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/elb" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" +) + +type awsClient struct { +} + +func (c *awsClient) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { + return &ec2.DescribeImagesOutput{ + Images: []*ec2.Image{ + { + ImageId: aws.String("ami-a9acbbd6"), + }, + }, + }, nil +} + +func (c *awsClient) DescribeVpcs(input *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) { + return &ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{ + { + VpcId: aws.String("vpc-32677e0e794418639"), + }, + }, + }, nil +} + +func (c *awsClient) DescribeSubnets(input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) { + return &ec2.DescribeSubnetsOutput{ + Subnets: []*ec2.Subnet{ + { + SubnetId: aws.String("subnet-28fddb3c45cae61b5"), + }, + }, + }, nil +} + +func (c *awsClient) DescribeSecurityGroups(input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) { + return &ec2.DescribeSecurityGroupsOutput{ + SecurityGroups: []*ec2.SecurityGroup{ + { + GroupId: aws.String("sg-05acc3c38a35ce63b"), + }, + }, + }, nil +} + +func (c *awsClient) RunInstances(input *ec2.RunInstancesInput) (*ec2.Reservation, error) { + return &ec2.Reservation{ + Instances: []*ec2.Instance{ + { + ImageId: aws.String("ami-a9acbbd6"), + InstanceId: aws.String("i-02fcb933c5da7085c"), + State: &ec2.InstanceState{ + Code: aws.Int64(16), + }, + }, + }, + }, nil +} + +func (c *awsClient) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) { + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + { + Instances: []*ec2.Instance{ + { + ImageId: aws.String("ami-a9acbbd6"), + InstanceId: aws.String("i-02fcb933c5da7085c"), + State: &ec2.InstanceState{ + Name: aws.String("Running"), + Code: aws.Int64(16), + }, + LaunchTime: aws.Time(time.Now()), + }, + }, + }, + }, + }, nil +} + +func (c *awsClient) TerminateInstances(input *ec2.TerminateInstancesInput) (*ec2.TerminateInstancesOutput, error) { + // Feel free to extend the returned values + return &ec2.TerminateInstancesOutput{}, nil +} + +func (c *awsClient) RegisterInstancesWithLoadBalancer(input *elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) { + // Feel free to extend the returned values + return &elb.RegisterInstancesWithLoadBalancerOutput{}, nil +} + +// NewClient creates our client wrapper object for the actual AWS clients we use. +// For authentication the underlying clients will use either the cluster AWS credentials +// secret if defined (i.e. in the root cluster), +// otherwise the IAM profile of the master where the actuator will run. (target clusters) +func NewClient(kubeClient kubernetes.Interface, secretName, namespace, region string) (client.Client, error) { + return &awsClient{}, nil +} diff --git a/cloud/aws/logging/logging.go b/cloud/aws/logging/logging.go new file mode 100644 index 0000000000..3abd3fc753 --- /dev/null +++ b/cloud/aws/logging/logging.go @@ -0,0 +1,49 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logging + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clusterapi "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + + log "github.com/sirupsen/logrus" +) + +// WithMachineSet expands a logger's context to include info about the given machineset. +func WithMachineSet(logger log.FieldLogger, machineSet *clusterapi.MachineSet) log.FieldLogger { + return WithGenericObject(logger, "machineset", machineSet) +} + +// WithCluster expands a logger's context to include info about the given cluster. +func WithCluster(logger log.FieldLogger, cluster *clusterapi.Cluster) log.FieldLogger { + return WithGenericObject(logger, "cluster", cluster) +} + +// WithGenericObject expands a logger's context to include info about the given object. +func WithGenericObject(logger log.FieldLogger, objectType string, obj metav1.Object) log.FieldLogger { + return logger.WithField(objectType, fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName())) +} + +// WithMachine expands a logger's context to include info about the given machine. +func WithMachine(logger log.FieldLogger, machine *clusterv1.Machine) log.FieldLogger { + return WithGenericObject(logger, "machine", machine) +} diff --git a/cloud/aws/providerconfig/types.go b/cloud/aws/providerconfig/types.go index 5b9070213a..16b56cc324 100644 --- a/cloud/aws/providerconfig/types.go +++ b/cloud/aws/providerconfig/types.go @@ -14,25 +14,159 @@ package providerconfig import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// AWSMachineProviderConfig is the type that will be embedded in a Machine.Spec.ProviderConfig field +// for an AWS instance. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSMachineProviderConfig struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta + + // AMI is the reference to the AMI from which to create the machine instance. + AMI AWSResourceReference + + // InstanceType is the type of instance to create. Example: m4.xlarge + InstanceType string + + // Tags is the set of tags to add to apply to an instance, in addition to the ones + // added by default by the actuator. These tags are additive. The actuator will ensure + // these tags are present, but will not remove any other tags that may exist on the + // instance. + Tags []TagSpecification + + // IAMInstanceProfile is a reference to an IAM role to assign to the instance + IAMInstanceProfile *AWSResourceReference + + // UserDataSecret contains a local reference to a secret that contains the + // UserData to apply to the instance + UserDataSecret *corev1.LocalObjectReference + + // CredentialsSecret is a reference to the secret with AWS credentials. Otherwise, defaults to permissions + // provided by attached IAM role where the actuator is running. + CredentialsSecret *corev1.LocalObjectReference + + // KeyName is the name of the KeyPair to use for SSH + KeyName *string + + // DeviceIndex is the index of the device on the instance for the network interface attachment. + // Defaults to 0. + DeviceIndex int64 + + // PublicIP specifies whether the instance should get a public IP. If not present, + // it should use the default of its subnet. + PublicIP *bool + + // SecurityGroups is an array of references to security groups that should be applied to the + // instance. + SecurityGroups []AWSResourceReference + + // Subnet is a reference to the subnet to use for this instance + Subnet AWSResourceReference + + // Placement specifies where to create the instance in AWS + Placement Placement + + // LoadBalancerIDs is the IDs of the load balancers to which the new instance + // should be added once it is created. + LoadBalancerIDs []AWSResourceReference +} + +// AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters. +// Only one of ID, ARN or Filters may be specified. Specifying more than one will result in +// a validation error. +type AWSResourceReference struct { + // ID of resource + ID *string + + // ARN of resource + ARN *string + + // Filters is a set of filters used to identify a resource + Filters []Filter +} + +// Placement indicates where to create the instance in AWS +type Placement struct { + // Region is the region to use to create the instance + Region string + + // AvailabilityZone is the availability zone of the instance + AvailabilityZone string +} + +// Filter is a filter used to identify an AWS resource +type Filter struct { + // Name of the filter. Filter names are case-sensitive. + Name string + + // Values includes one or more filter values. Filter values are case-sensitive. + Values []string +} + +// TagSpecification is the name/value pair for a tag +type TagSpecification struct { + // Name of the tag + Name string + + // Value of the tag + Value string } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSClusterProviderConfig struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta } +// AWSMachineProviderStatus is the type that will be embedded in a Machine.Status.ProviderStatus field. +// It containsk AWS-specific status information. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSMachineProviderStatus struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta + + // InstanceID is the instance ID of the machine created in AWS + InstanceID *string + + // InstanceState is the state of the AWS instance for this machine + InstanceState *string + + // Conditions is a set of conditions associated with the Machine to indicate + // errors or other status + Conditions []AWSMachineProviderCondition +} + +// AWSMachineProviderConditionType is a valid value for AWSMachineProviderCondition.Type +type AWSMachineProviderConditionType string + +// Valid conditions for an AWS machine instance +const ( + // MachineCreated indicates whether the machine has been created or not. If not, + // it should include a reason and message for the failure. + MachineCreated AWSMachineProviderConditionType = "MachineCreated" +) + +// AWSMachineProviderCondition is a condition in a AWSMachineProviderStatus +type AWSMachineProviderCondition struct { + // Type is the type of the condition. + Type AWSMachineProviderConditionType + // Status is the status of the condition. + Status corev1.ConditionStatus + // LastProbeTime is the last time we probed the condition. + // +optional + LastProbeTime metav1.Time + // LastTransitionTime is the last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time + // Reason is a unique, one-word, CamelCase reason for the condition's last transition. + // +optional + Reason string + // Message is a human-readable message indicating details about last transition. + // +optional + Message string } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSClusterProviderStatus struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta } diff --git a/cloud/aws/providerconfig/v1alpha1/register.go b/cloud/aws/providerconfig/v1alpha1/register.go index ebbe81e10b..3b54fdeb4a 100644 --- a/cloud/aws/providerconfig/v1alpha1/register.go +++ b/cloud/aws/providerconfig/v1alpha1/register.go @@ -41,11 +41,19 @@ var ( AddToScheme = localSchemeBuilder.AddToScheme ) +var ( + // Codecs for creating a server config + Scheme, _ = NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) + Encoder, _ = newEncoder(&Codecs) +) + func init() { localSchemeBuilder.Register(addKnownTypes) } func addKnownTypes(scheme *runtime.Scheme) error { + fmt.Printf("Registering v1alpha1 types\n") scheme.AddKnownTypes(SchemeGroupVersion, &AWSMachineProviderConfig{}, ) @@ -62,6 +70,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { } func NewScheme() (*runtime.Scheme, error) { + fmt.Printf("New Scheme v1alpha1 types\n") scheme := runtime.NewScheme() if err := AddToScheme(scheme); err != nil { return nil, err @@ -69,6 +78,7 @@ func NewScheme() (*runtime.Scheme, error) { if err := providerconfig.AddToScheme(scheme); err != nil { return nil, err } + addKnownTypes(scheme) return scheme, nil } diff --git a/cloud/aws/providerconfig/v1alpha1/types.go b/cloud/aws/providerconfig/v1alpha1/types.go index 343f423d3d..c495d65284 100644 --- a/cloud/aws/providerconfig/v1alpha1/types.go +++ b/cloud/aws/providerconfig/v1alpha1/types.go @@ -1,38 +1,213 @@ -// Copyright © 2018 The Kubernetes Authors. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// AWSMachineProviderConfig is the type that will be embedded in a Machine.Spec.ProviderConfig field +// for an AWS instance. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSMachineProviderConfig struct { metav1.TypeMeta `json:",inline"` + + // AMI is the reference to the AMI from which to create the machine instance. + AMI AWSResourceReference `json:"ami"` + + // InstanceType is the type of instance to create. Example: m4.xlarge + InstanceType string `json:"instanceType"` + + // Tags is the set of tags to add to apply to an instance, in addition to the ones + // added by default by the actuator. These tags are additive. The actuator will ensure + // these tags are present, but will not remove any other tags that may exist on the + // instance. + Tags []TagSpecification `json:"tags"` + + // IAMInstanceProfile is a reference to an IAM role to assign to the instance + IAMInstanceProfile *AWSResourceReference `json:"iamInstanceProfile"` + + // UserDataSecret contains a local reference to a secret that contains the + // UserData to apply to the instance + UserDataSecret *corev1.LocalObjectReference `json:"userDataSecret"` + + // CredentialsSecret is a reference to the secret with AWS credentials. Otherwise, defaults to permissions + // provided by attached IAM role where the actuator is running. + CredentialsSecret *corev1.LocalObjectReference `json:"credentialsSecret"` + + // KeyName is the name of the KeyPair to use for SSH + KeyName *string `json:"keyName"` + + // DeviceIndex is the index of the device on the instance for the network interface attachment. + // Defaults to 0. + DeviceIndex int64 `json:"deviceIndex"` + + // PublicIP specifies whether the instance should get a public IP. If not present, + // it should use the default of its subnet. + PublicIP *bool `json:"publicIp"` + + // SecurityGroups is an array of references to security groups that should be applied to the + // instance. + SecurityGroups []AWSResourceReference `json:"securityGroups"` + + // Subnet is a reference to the subnet to use for this instance + Subnet AWSResourceReference `json:"subnet"` + + // Placement specifies where to create the instance in AWS + Placement Placement `json:"placement"` + + // LoadBalancerIDs is the IDs of the load balancers to which the new instance + // should be added once it is created. + LoadBalancerIDs []AWSResourceReference `json:"loadBalancerIds"` +} + +// AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters +type AWSResourceReference struct { + // ID of resource + ID *string `json:"id"` + + // ARN of resource + ARN *string `json:"arn"` + + // Filters is a set of filters used to identify a resource + Filters []Filter `json:"filters"` +} + +// Placement indicates where to create the instance in AWS +type Placement struct { + // Region is the region to use to create the instance + Region string `json:"region"` + + // AvailabilityZone is the availability zone of the instance + AvailabilityZone string `json:"availabilityZone"` +} + +// Filter is a filter used to identify an AWS resource +type Filter struct { + // Name of the filter + Name string `json:"name"` + + // Values includes one or more filter values + Values []string `json:"values"` +} + +// TagSpecification is the name/value pair for a tag +type TagSpecification struct { + // Name of the tag + Name string `json:"name"` + + // Value of the tag + Value string `json:"value"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type AWSClusterProviderStatus struct { + metav1.TypeMeta `json:",inline"` } +// Annotation constants +const ( + // ClusterNameLabel is the label that a machineset must have to identify the + // cluster to which it belongs. + ClusterNameLabel = "sigs.k8s.io/cluster-api-cluster" + MachineRoleLabel = "sigs.k8s.io/cluster-api-machine-role" + MachineTypeLabel = "sigs.k8s.io/cluster-api-machine-type" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// AWSClusterProviderConfig is the AWS specific cluster specification stored in the ProviderConfig of an AWS cluster.k8s.io.Cluster. type AWSClusterProviderConfig struct { + // +optional metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // KeyPairName is the name of the AWS key pair to use for SSH access to EC2 + // instances in this cluster + KeyPairName string `json:"keyPairName"` + + // Region specifies the AWS region where the cluster will be created + Region string `json:"region"` + + // VPCName specifies the name of the VPC to associate with the cluster. + // If a value is specified, a VPC will be created with that name if it + // does not already exist in the cloud provider. If it does exist, the + // existing VPC will be used. + // If no name is specified, a VPC name will be generated using the + // cluster name and created in the cloud provider. + // +optional + VPCName string `json:"vpcName,omitempty"` } +// AWSMachineProviderStatus is the type that will be embedded in a Machine.Status.ProviderStatus field. +// It containsk AWS-specific status information. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type AWSMachineProviderStatus struct { metav1.TypeMeta `json:",inline"` + + // InstanceID is the instance ID of the machine created in AWS + InstanceID *string `json:"instanceId"` + + // InstanceState is the state of the AWS instance for this machine + InstanceState *string `json:"instanceState"` + + // Conditions is a set of conditions associated with the Machine to indicate + // errors or other status + Conditions []AWSMachineProviderCondition `json:"conditions"` } -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -type AWSClusterProviderStatus struct { - metav1.TypeMeta `json:",inline"` +// AWSMachineProviderConditionType is a valid value for AWSMachineProviderCondition.Type +type AWSMachineProviderConditionType string + +// Valid conditions for an AWS machine instance +const ( + // MachineCreated indicates whether the machine has been created or not. If not, + // it should include a reason and message for the failure. + MachineCreated AWSMachineProviderConditionType = "MachineCreated" +) + +// AWSMachineProviderCondition is a condition in a AWSMachineProviderStatus +type AWSMachineProviderCondition struct { + // Type is the type of the condition. + Type AWSMachineProviderConditionType `json:"type"` + // Status is the status of the condition. + Status corev1.ConditionStatus `json:"status"` + // LastProbeTime is the last time we probed the condition. + // +optional + LastProbeTime metav1.Time `json:"lastProbeTime"` + // LastTransitionTime is the last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime"` + // Reason is a unique, one-word, CamelCase reason for the condition's last transition. + // +optional + Reason string `json:"reason"` + // Message is a human-readable message indicating details about last transition. + // +optional + Message string `json:"message"` } + +// NodeType is the type of the Node +type NodeType string + +const ( + // NodeTypeMaster is a node that is a master in the cluster + NodeTypeMaster NodeType = "Master" + // NodeTypeCompute is a node that is a compute node in the cluster + NodeTypeCompute NodeType = "Compute" +) diff --git a/cloud/aws/providerconfig/v1alpha1/zz_generated.deepcopy.go b/cloud/aws/providerconfig/v1alpha1/zz_generated.deepcopy.go index adcce680be..c3f584119d 100644 --- a/cloud/aws/providerconfig/v1alpha1/zz_generated.deepcopy.go +++ b/cloud/aws/providerconfig/v1alpha1/zz_generated.deepcopy.go @@ -17,6 +17,7 @@ package v1alpha1 import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -24,6 +25,7 @@ import ( func (in *AWSClusterProviderConfig) DeepCopyInto(out *AWSClusterProviderConfig) { *out = *in out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) return } @@ -70,10 +72,75 @@ func (in *AWSClusterProviderStatus) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSMachineProviderCondition) DeepCopyInto(out *AWSMachineProviderCondition) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineProviderCondition. +func (in *AWSMachineProviderCondition) DeepCopy() *AWSMachineProviderCondition { + if in == nil { + return nil + } + out := new(AWSMachineProviderCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AWSMachineProviderConfig) DeepCopyInto(out *AWSMachineProviderConfig) { *out = *in out.TypeMeta = in.TypeMeta + in.AMI.DeepCopyInto(&out.AMI) + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]TagSpecification, len(*in)) + copy(*out, *in) + } + if in.IAMInstanceProfile != nil { + in, out := &in.IAMInstanceProfile, &out.IAMInstanceProfile + *out = new(AWSResourceReference) + (*in).DeepCopyInto(*out) + } + if in.UserDataSecret != nil { + in, out := &in.UserDataSecret, &out.UserDataSecret + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.CredentialsSecret != nil { + in, out := &in.CredentialsSecret, &out.CredentialsSecret + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.KeyName != nil { + in, out := &in.KeyName, &out.KeyName + *out = new(string) + **out = **in + } + if in.PublicIP != nil { + in, out := &in.PublicIP, &out.PublicIP + *out = new(bool) + **out = **in + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]AWSResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Subnet.DeepCopyInto(&out.Subnet) + out.Placement = in.Placement + if in.LoadBalancerIDs != nil { + in, out := &in.LoadBalancerIDs, &out.LoadBalancerIDs + *out = make([]AWSResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -99,6 +166,23 @@ func (in *AWSMachineProviderConfig) DeepCopyObject() runtime.Object { func (in *AWSMachineProviderStatus) DeepCopyInto(out *AWSMachineProviderStatus) { *out = *in out.TypeMeta = in.TypeMeta + if in.InstanceID != nil { + in, out := &in.InstanceID, &out.InstanceID + *out = new(string) + **out = **in + } + if in.InstanceState != nil { + in, out := &in.InstanceState, &out.InstanceState + *out = new(string) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]AWSMachineProviderCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -119,3 +203,89 @@ func (in *AWSMachineProviderStatus) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSResourceReference) DeepCopyInto(out *AWSResourceReference) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } + if in.ARN != nil { + in, out := &in.ARN, &out.ARN + *out = new(string) + **out = **in + } + if in.Filters != nil { + in, out := &in.Filters, &out.Filters + *out = make([]Filter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSResourceReference. +func (in *AWSResourceReference) DeepCopy() *AWSResourceReference { + if in == nil { + return nil + } + out := new(AWSResourceReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Filter) DeepCopyInto(out *Filter) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. +func (in *Filter) DeepCopy() *Filter { + if in == nil { + return nil + } + out := new(Filter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Placement) DeepCopyInto(out *Placement) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Placement. +func (in *Placement) DeepCopy() *Placement { + if in == nil { + return nil + } + out := new(Placement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TagSpecification) DeepCopyInto(out *TagSpecification) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TagSpecification. +func (in *TagSpecification) DeepCopy() *TagSpecification { + if in == nil { + return nil + } + out := new(TagSpecification) + in.DeepCopyInto(out) + return out +} diff --git a/cloud/aws/providerconfig/zz_generated.deepcopy.go b/cloud/aws/providerconfig/zz_generated.deepcopy.go index a60b6d64bd..dd10c81d02 100644 --- a/cloud/aws/providerconfig/zz_generated.deepcopy.go +++ b/cloud/aws/providerconfig/zz_generated.deepcopy.go @@ -17,6 +17,7 @@ package providerconfig import ( + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -70,10 +71,75 @@ func (in *AWSClusterProviderStatus) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSMachineProviderCondition) DeepCopyInto(out *AWSMachineProviderCondition) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSMachineProviderCondition. +func (in *AWSMachineProviderCondition) DeepCopy() *AWSMachineProviderCondition { + if in == nil { + return nil + } + out := new(AWSMachineProviderCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AWSMachineProviderConfig) DeepCopyInto(out *AWSMachineProviderConfig) { *out = *in out.TypeMeta = in.TypeMeta + in.AMI.DeepCopyInto(&out.AMI) + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]TagSpecification, len(*in)) + copy(*out, *in) + } + if in.IAMInstanceProfile != nil { + in, out := &in.IAMInstanceProfile, &out.IAMInstanceProfile + *out = new(AWSResourceReference) + (*in).DeepCopyInto(*out) + } + if in.UserDataSecret != nil { + in, out := &in.UserDataSecret, &out.UserDataSecret + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.CredentialsSecret != nil { + in, out := &in.CredentialsSecret, &out.CredentialsSecret + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.KeyName != nil { + in, out := &in.KeyName, &out.KeyName + *out = new(string) + **out = **in + } + if in.PublicIP != nil { + in, out := &in.PublicIP, &out.PublicIP + *out = new(bool) + **out = **in + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]AWSResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Subnet.DeepCopyInto(&out.Subnet) + out.Placement = in.Placement + if in.LoadBalancerIDs != nil { + in, out := &in.LoadBalancerIDs, &out.LoadBalancerIDs + *out = make([]AWSResourceReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -99,6 +165,23 @@ func (in *AWSMachineProviderConfig) DeepCopyObject() runtime.Object { func (in *AWSMachineProviderStatus) DeepCopyInto(out *AWSMachineProviderStatus) { *out = *in out.TypeMeta = in.TypeMeta + if in.InstanceID != nil { + in, out := &in.InstanceID, &out.InstanceID + *out = new(string) + **out = **in + } + if in.InstanceState != nil { + in, out := &in.InstanceState, &out.InstanceState + *out = new(string) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]AWSMachineProviderCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } @@ -119,3 +202,89 @@ func (in *AWSMachineProviderStatus) DeepCopyObject() runtime.Object { } return nil } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AWSResourceReference) DeepCopyInto(out *AWSResourceReference) { + *out = *in + if in.ID != nil { + in, out := &in.ID, &out.ID + *out = new(string) + **out = **in + } + if in.ARN != nil { + in, out := &in.ARN, &out.ARN + *out = new(string) + **out = **in + } + if in.Filters != nil { + in, out := &in.Filters, &out.Filters + *out = make([]Filter, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWSResourceReference. +func (in *AWSResourceReference) DeepCopy() *AWSResourceReference { + if in == nil { + return nil + } + out := new(AWSResourceReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Filter) DeepCopyInto(out *Filter) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. +func (in *Filter) DeepCopy() *Filter { + if in == nil { + return nil + } + out := new(Filter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Placement) DeepCopyInto(out *Placement) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Placement. +func (in *Placement) DeepCopy() *Placement { + if in == nil { + return nil + } + out := new(Placement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TagSpecification) DeepCopyInto(out *TagSpecification) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TagSpecification. +func (in *TagSpecification) DeepCopy() *TagSpecification { + if in == nil { + return nil + } + out := new(TagSpecification) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/cluster-controller/Makefile b/cmd/cluster-controller/Makefile index abef2e8f40..eaef2680d6 100644 --- a/cmd/cluster-controller/Makefile +++ b/cmd/cluster-controller/Makefile @@ -21,7 +21,7 @@ NAME = aws-cluster-controller TAG = 0.0.1 image: - docker build -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../.. + imagebuilder -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../.. push: image docker push "$(PREFIX)/$(NAME):$(TAG)" diff --git a/cmd/machine-controller/Makefile b/cmd/machine-controller/Makefile index b80ef959b9..1d142b73a6 100644 --- a/cmd/machine-controller/Makefile +++ b/cmd/machine-controller/Makefile @@ -21,7 +21,7 @@ NAME = aws-machine-controller TAG = 0.0.1 image: - docker build -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../.. + imagebuilder -t "$(PREFIX)/$(NAME):$(TAG)" -f ./Dockerfile ../.. push: image docker push "$(PREFIX)/$(NAME):$(TAG)" diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 304cdfd2af..71f1503a76 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -18,20 +18,36 @@ package main import ( "flag" + "os" "github.com/golang/glog" + "github.com/kubernetes-incubator/apiserver-builder/pkg/controller" + log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "k8s.io/apiserver/pkg/util/logs" + "k8s.io/client-go/kubernetes" + machineactuator "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/actuators/machine" + awsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" + "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset" "sigs.k8s.io/cluster-api/pkg/controller/config" + "sigs.k8s.io/cluster-api/pkg/controller/machine" + "sigs.k8s.io/cluster-api/pkg/controller/sharedinformers" +) - "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/controllers/machine" - "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/controllers/machine/options" +var ( + logLevel string ) func init() { config.ControllerConfig.AddFlags(pflag.CommandLine) + pflag.CommandLine.StringVar(&logLevel, "log-level", defaultLogLevel, "Log level (debug,info,warn,error,fatal)") } +const ( + controllerLogName = "awsMachine" + defaultLogLevel = "info" +) + func main() { // the following line exists to make glog happy, for more information, see: https://github.com/kubernetes/kubernetes/issues/17162 flag.CommandLine.Parse([]string{}) @@ -40,8 +56,40 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - machineServer := options.NewServer() - if err := machine.Run(machineServer); err != nil { - glog.Errorf("Failed to start the machine controller. Err: %v", err) + config, err := controller.GetConfig(config.ControllerConfig.Kubeconfig) + if err != nil { + glog.Fatalf("Could not create Config for talking to the apiserver: %v", err) + } + + client, err := clientset.NewForConfig(config) + if err != nil { + glog.Fatalf("Could not create client for talking to the apiserver: %v", err) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + glog.Fatalf("Could not create kubernetes client to talk to the apiserver: %v", err) } + + log.SetOutput(os.Stdout) + if lvl, err := log.ParseLevel(logLevel); err != nil { + log.Panic(err) + } else { + log.SetLevel(lvl) + } + + logger := log.WithField("controller", controllerLogName) + + actuator, err := machineactuator.NewActuator(kubeClient, client, logger, awsclient.NewClient) + if err != nil { + glog.Fatalf("Could not create AWS machine actuator: %v", err) + } + + shutdown := make(chan struct{}) + si := sharedinformers.NewSharedInformers(config, shutdown) + // If this doesn't compile, the code generator probably + // overwrote the customized NewMachineController function. + c := machine.NewMachineController(config, si, actuator) + c.Run(shutdown) + select {} } diff --git a/examples/machine.yaml b/examples/machine.yaml new file mode 100644 index 0000000000..ea2fffa21d --- /dev/null +++ b/examples/machine.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: Machine +metadata: + name: aws-actuator-testing-machine + namespace: default + generateName: vs-master- + labels: + sigs.k8s.io/cluster-api-cluster: tb-asg-35 + sigs.k8s.io/cluster-api-machine-role: infra + sigs.k8s.io/cluster-api-machine-type: master +spec: + providerConfig: + value: + apiVersion: awsproviderconfig/v1alpha1 + kind: AWSMachineProviderConfig + ami: + id: ami-a9acbbd6 + credentialsSecret: + name: aws-credentials-secret + instanceType: m4.xlarge + placement: + region: us-east-1 + availabilityZone: us-east-1a + subnet: + id: subnet-0e56b13a64ff8a941 + iamInstanceProfile: + id: openshift_master_launch_instances + keyName: libra + userDataSecret: + name: aws-actuator-user-data-secret + tags: + - name: openshift-node-group-config + value: node-config-master + - name: host-type + value: master + - name: sub-host-type + value: default + securityGroups: + - id: sg-00868b02fbe29de17 + - id: sg-0a4658991dc5eb40a + - id: sg-009a70e28fa4ba84e + - id: sg-07323d56fb932c84c + - id: sg-08b1ffd32874d59a2 + publicIP: true + versions: + kubelet: 1.10.1 + controlPlane: 1.10.1 diff --git a/test/integration/create_update_delete_test.go b/test/integration/create_update_delete_test.go new file mode 100644 index 0000000000..e59ad8274f --- /dev/null +++ b/test/integration/create_update_delete_test.go @@ -0,0 +1,146 @@ +package integration + +import ( + "io/ioutil" + "os" + "path" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + + "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/fake" + + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + machineactuator "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/actuators/machine" + awsclient "sigs.k8s.io/cluster-api-provider-aws/cloud/aws/client" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + + "github.com/ghodss/yaml" +) + +const ( + controllerLogName = "awsMachine" + defaultLogLevel = "info" + + defaultNamespace = "default" + awsCredentialsSecretName = "aws-credentials-secret" + userDataSecretName = "aws-actuator-user-data-secret" + + clusterName = "tb-asg-35" +) + +const userDataBlob = `#cloud-config +write_files: +- path: /root/node_bootstrap/node_settings.yaml + owner: 'root:root' + permissions: '0640' + content: | + node_config_name: node-config-master +runcmd: +- [ cat, /root/node_bootstrap/node_settings.yaml] +` + +func testMachineAPIResources(clusterID string) (*clusterv1.Machine, *clusterv1.Cluster, *apiv1.Secret, *apiv1.Secret, error) { + machine := &clusterv1.Machine{} + + bytes, err := ioutil.ReadFile(path.Join(os.Getenv("GOPATH"), "/src/sigs.k8s.io/cluster-api-provider-aws/examples/machine.yaml")) + if err != nil { + return nil, nil, nil, nil, err + } + + if err = yaml.Unmarshal(bytes, &machine); err != nil { + return nil, nil, nil, nil, err + } + + awsCredentialsSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: awsCredentialsSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + awsclient.AwsCredsSecretIDKey: []byte(os.Getenv("AWS_ACCESS_KEY_ID")), + awsclient.AwsCredsSecretAccessKey: []byte(os.Getenv("AWS_SECRET_ACCESS_KEY")), + }, + } + + userDataSecret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: defaultNamespace, + }, + Data: map[string][]byte{ + "user-data": []byte(userDataBlob), + }, + } + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterID, + Namespace: defaultNamespace, + }, + } + + return machine, cluster, awsCredentialsSecret, userDataSecret, nil +} + +func TestCreateAndDeleteMachine(t *testing.T) { + + // kube client is needed to fetch aws credentials: + // - kubeClient.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + // cluster client for updating machine statues + // - clusterClient.ClusterV1alpha1().Machines(machineCopy.Namespace).UpdateStatus(machineCopy) + + if os.Getenv("GOPATH") == "" { + t.Fatalf("GOPATH not set") + } + + machine, cluster, awsCredentialsSecret, userDataSecret, err := testMachineAPIResources(clusterName) + if err != nil { + t.Fatal(err) + } + + fakeKubeClient := kubernetesfake.NewSimpleClientset(awsCredentialsSecret, userDataSecret) + fakeClient := fake.NewSimpleClientset(machine) + logger := log.WithField("controller", controllerLogName) + + actuator, err := machineactuator.NewActuator(fakeKubeClient, fakeClient, logger, awsclient.NewClient) + if err != nil { + t.Fatalf("Could not create Openstack machine actuator: %v", err) + } + + // Create a machine + if err := actuator.Create(cluster, machine); err != nil { + t.Errorf("Unable to create instance for machine: %v", err) + } + + // Get the machine + if exists, err := actuator.Exists(cluster, machine); err != nil || !exists { + t.Errorf("Instance for %v does not exists: %v", strings.Join([]string{machine.Namespace, machine.Name}, "/"), err) + } else { + t.Logf("Instance for %v exists", strings.Join([]string{machine.Namespace, machine.Name}, "/")) + } + + // TODO(jchaloup): Wait until the machine is ready + + // Update a machine + if err := actuator.Update(cluster, machine); err != nil { + t.Errorf("Unable to create instance for machine: %v", err) + } + + // Get the machine + if exists, err := actuator.Exists(cluster, machine); err != nil || !exists { + t.Errorf("Instance for %v does not exists: %v", strings.Join([]string{machine.Namespace, machine.Name}, "/"), err) + } else { + t.Logf("Instance for %v exists", strings.Join([]string{machine.Namespace, machine.Name}, "/")) + } + + // Delete a machine + if err := actuator.Delete(cluster, machine); err != nil { + t.Errorf("Unable to delete instance for machine: %v", err) + } +}