diff --git a/Makefile b/Makefile index 4acb37f2c5..00c56b6055 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GO111MODULE = on export GO111MODULE -GOFLAGS += -mod=vendor +GOFLAGS ?= -mod=vendor export GOFLAGS GOPROXY ?= export GOPROXY @@ -56,7 +56,9 @@ generate: hack/goimports.sh . .PHONY: test -test: unit +test: ## Run tests + @echo -e "\033[32mTesting...\033[0m" + $(DOCKER_CMD) hack/ci-test.sh bin: @mkdir $@ @@ -68,7 +70,6 @@ build: ## build binaries $(DOCKER_CMD) go build $(GOGCFLAGS) -o bin/manager -ldflags '-extldflags "-static"' \ "$(REPO_PATH)/vendor/github.com/openshift/machine-api-operator/cmd/machineset" - aws-actuator: $(DOCKER_CMD) go build $(GOGCFLAGS) -o bin/aws-actuator sigs.k8s.io/cluster-api-provider-aws/cmd/aws-actuator @@ -96,7 +97,6 @@ unit: # Run unit test test-e2e: ## Run e2e tests hack/e2e.sh - .PHONY: lint lint: ## Go lint your code hack/go-lint.sh -min_confidence 0.3 $$(go list -f '{{ .ImportPath }}' ./... | grep -v -e 'sigs.k8s.io/cluster-api-provider-aws/test' -e 'sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/client/mock') diff --git a/hack/ci-test.sh b/hack/ci-test.sh new file mode 100755 index 0000000000..0240578d50 --- /dev/null +++ b/hack/ci-test.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE}")/.. + +cd $REPO_ROOT && \ + source ./hack/fetch-ext-bins.sh && \ + fetch_tools && \ + setup_envs && \ + make unit diff --git a/hack/fetch-ext-bins.sh b/hack/fetch-ext-bins.sh new file mode 100644 index 0000000000..07bc1338d1 --- /dev/null +++ b/hack/fetch-ext-bins.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# Enable tracing in this script off by setting the TRACE variable in your +# environment to any value: +# +# $ TRACE=1 test.sh +TRACE=${TRACE:-""} +if [ -n "$TRACE" ]; then + set -x +fi + +# TODO: update to k8s 1.17 +k8s_version=1.16.4 +goarch=amd64 +goos="unknown" + +if [[ "$OSTYPE" == "linux"* ]]; then + goos="linux" +elif [[ "$OSTYPE" == "darwin"* ]]; then + goos="darwin" +fi + +if [[ "$goos" == "unknown" ]]; then + echo "OS '$OSTYPE' not supported. Aborting." >&2 + exit 1 +fi + +# Turn colors in this script off by setting the NO_COLOR variable in your +# environment to any value: +# +# $ NO_COLOR=1 test.sh +NO_COLOR=${NO_COLOR:-""} +if [ -z "$NO_COLOR" ]; then + header=$'\e[1;33m' + reset=$'\e[0m' +else + header='' + reset='' +fi + +function header_text { + echo "$header$*$reset" +} + +rc=0 +tmp_root=/tmp + +kb_root_dir=$tmp_root/kubebuilder +kb_orig=$(pwd) + +# Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable +# in your environment to any value: +# +# $ SKIP_FETCH_TOOLS=1 ./fetch_ext_bins.sh +# +# If you skip fetching tools, this script will use the tools already on your +# machine, but rebuild the kubebuilder and kubebuilder-bin binaries. +SKIP_FETCH_TOOLS=${SKIP_FETCH_TOOLS:-""} + +function prepare_staging_dir { + header_text "preparing staging dir" + + if [ -z "$SKIP_FETCH_TOOLS" ]; then + rm -rf "$kb_root_dir" + else + rm -f "$kb_root_dir/kubebuilder/bin/kubebuilder" + rm -f "$kb_root_dir/kubebuilder/bin/kubebuilder-gen" + rm -f "$kb_root_dir/kubebuilder/bin/vendor.tar.gz" + fi +} + +# fetch k8s API gen tools and make it available under kb_root_dir/bin. +function fetch_tools { + if [ -n "$SKIP_FETCH_TOOLS" ]; then + return 0 + fi + + header_text "fetching tools" + kb_tools_archive_name="kubebuilder-tools-$k8s_version-$goos-$goarch.tar.gz" + kb_tools_download_url="https://storage.googleapis.com/kubebuilder-tools/$kb_tools_archive_name" + + kb_tools_archive_path="$tmp_root/$kb_tools_archive_name" + if [ ! -f $kb_tools_archive_path ]; then + curl -fsL ${kb_tools_download_url} -o "$kb_tools_archive_path" + fi + + tar -zvxf "$kb_tools_archive_path" -C "$tmp_root/" +} + +function setup_envs { + header_text "setting up env vars" + + # Setup env vars + export PATH=/tmp/kubebuilder/bin:$PATH + export TEST_ASSET_KUBECTL=/tmp/kubebuilder/bin/kubectl + export TEST_ASSET_KUBE_APISERVER=/tmp/kubebuilder/bin/kube-apiserver + export TEST_ASSET_ETCD=/tmp/kubebuilder/bin/etcd + export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=10m + export NO_DOCKER=1 +} diff --git a/pkg/actuators/machine/actuator.go b/pkg/actuators/machine/actuator.go index 43d7c42e33..41858f955b 100644 --- a/pkg/actuators/machine/actuator.go +++ b/pkg/actuators/machine/actuator.go @@ -633,7 +633,7 @@ func (a *Actuator) patchMachine(ctx context.Context, machine *machinev1.Machine, //Patch status if err := a.client.Status().Patch(ctx, machine, machineToBePatched); err != nil { - klog.Errorf("Failed to update machine %q: %v", machine.GetName(), err) + klog.Errorf("Failed to update machine status %q: %v", machine.GetName(), err) return err } return nil diff --git a/pkg/actuators/machine/actuator_test.go b/pkg/actuators/machine/actuator_test.go index 1f268e2bcf..128334c140 100644 --- a/pkg/actuators/machine/actuator_test.go +++ b/pkg/actuators/machine/actuator_test.go @@ -4,28 +4,32 @@ import ( "bytes" "context" "fmt" + "path/filepath" "strings" "testing" "time" - "k8s.io/utils/pointer" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" machinev1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" machineapierrors "github.com/openshift/machine-api-operator/pkg/controller/machine" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" providerconfigv1 "sigs.k8s.io/cluster-api-provider-aws/pkg/apis/awsproviderconfig/v1beta1" awsclient "sigs.k8s.io/cluster-api-provider-aws/pkg/client" mockaws "sigs.k8s.io/cluster-api-provider-aws/pkg/client/mock" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" ) func init() { @@ -229,14 +233,6 @@ func TestActuator(t *testing.T) { t.Fatalf("unable to build codec: %v", err) } - getMachineStatus := func(objectClient client.Client, machine *machinev1.Machine) (*providerconfigv1.AWSMachineProviderStatus, error) { - machineStatus := &providerconfigv1.AWSMachineProviderStatus{} - if err := codec.DecodeProviderStatus(machine.Status.ProviderStatus, machineStatus); err != nil { - return nil, fmt.Errorf("error decoding machine provider status: %v", err) - } - return machineStatus, nil - } - machineInvalidProviderConfig := machine.DeepCopy() machineInvalidProviderConfig.Spec.ProviderSpec.Value = nil @@ -266,7 +262,7 @@ func TestActuator(t *testing.T) { createErr := actuator.Create(context.TODO(), machine) assert.NoError(t, createErr) - machineStatus, err := getMachineStatus(objectClient, machine) + machineStatus, err := getMachineStatus(codec, machine) if err != nil { t.Fatalf("Unable to get machine status: %v", err) } @@ -306,7 +302,7 @@ func TestActuator(t *testing.T) { createErr := actuator.Create(context.TODO(), machine) assert.Error(t, createErr) - machineStatus, err := getMachineStatus(objectClient, machine) + machineStatus, err := getMachineStatus(codec, machine) if err != nil { t.Fatalf("Unable to get machine status: %v", err) } @@ -1281,3 +1277,215 @@ func awsClientBuilderFunc(c awsclient.Client) awsclient.AwsClientBuilderFuncType return c, nil } } + +func TestPatchMachine(t *testing.T) { + // BEGIN: Set up test environment + g := NewWithT(t) + + testEnv := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + } + + var err error + cfg, err := testEnv.Start() + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cfg).ToNot(BeNil()) + defer func() { + g.Expect(testEnv.Stop()).To(Succeed()) + }() + + mgr, err := manager.New(cfg, manager.Options{ + Scheme: scheme.Scheme, + MetricsBindAddress: "0", + }) + g.Expect(err).ToNot(HaveOccurred()) + + actuator, err := initActuator(mgr, t) + g.Expect(err).ToNot(HaveOccurred()) + + doneMgr := make(chan struct{}) + go func() { + g.Expect(mgr.Start(doneMgr)).To(Succeed()) + }() + defer close(doneMgr) + + // END: setup test environment + + k8sClient := mgr.GetClient() + + awsCredentialsSecret := stubAwsCredentialsSecret() + g.Expect(k8sClient.Create(context.TODO(), awsCredentialsSecret)).To(Succeed()) + + userDataSecret := stubUserDataSecret() + g.Expect(k8sClient.Create(context.TODO(), userDataSecret)).To(Succeed()) + + codec, err := providerconfigv1.NewCodec() + if err != nil { + t.Fatalf("unable to build codec: %v", err) + } + + failedPhase := "Failed" + + testCases := []struct { + name string + mutate func(*machinev1.Machine) + expect func(*machinev1.Machine) error + }{ + { + name: "Test changing labels", + mutate: func(m *machinev1.Machine) { + m.ObjectMeta.Labels["testlabel"] = "test" + }, + expect: func(m *machinev1.Machine) error { + if m.ObjectMeta.Labels["testlabel"] != "test" { + return fmt.Errorf("label \"testlabel\" %q not equal expected \"test\"", m.ObjectMeta.Labels["test"]) + } + return nil + }, + }, + { + name: "Test setting phase", + mutate: func(m *machinev1.Machine) { + + m.Status.Phase = &failedPhase + }, + expect: func(m *machinev1.Machine) error { + if m.Status.Phase != nil && *m.Status.Phase == failedPhase { + return nil + } + return fmt.Errorf("phase is nil or not equal expected \"Failed\"") + }, + }, + { + name: "Test setting provider status", + mutate: func(m *machinev1.Machine) { + instanceID := "123" + instanceState := "running" + + providerStatus := &providerconfigv1.AWSMachineProviderStatus{ + InstanceID: &instanceID, + InstanceState: &instanceState, + } + + status, err := codec.EncodeProviderStatus(providerStatus) + if err != nil { + t.Fatal(err) + } + + m.Status.ProviderStatus = status + }, + expect: func(m *machinev1.Machine) error { + providerStatus, err := getMachineStatus(codec, m) + if err != nil { + return fmt.Errorf("unable to get provider status: %v", err) + } + + if providerStatus.InstanceID == nil || *providerStatus.InstanceID != "123" { + return fmt.Errorf("instanceID is nil or not equal expected \"123\"") + } + + if providerStatus.InstanceState == nil || *providerStatus.InstanceState != "running" { + return fmt.Errorf("instanceState is nil or not equal expected \"running\"") + } + + return nil + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + timeout := 10 * time.Second + gs := NewWithT(t) + + machine, err := stubMachine() + gs.Expect(err).ToNot(HaveOccurred()) + gs.Expect(machine).ToNot(BeNil()) + + ctx := context.TODO() + + // Create the machine + gs.Expect(k8sClient.Create(ctx, machine)).To(Succeed()) + + defer func() { + gs.Expect(k8sClient.Delete(ctx, machine)).To(Succeed()) + }() + + // Ensure the machine has synced to the cache + getMachine := func() error { + machineKey := types.NamespacedName{Namespace: machine.Namespace, Name: machine.Name} + return k8sClient.Get(ctx, machineKey, machine) + } + gs.Eventually(getMachine, timeout).Should(Succeed()) + + machineToBePatched := client.MergeFrom(machine.DeepCopy()) + tc.mutate(machine) + + // Patch the machine and check the expectation from the test case + gs.Expect(actuator.patchMachine(ctx, machine, machineToBePatched)).To(Succeed()) + checkExpectation := func() error { + if err := getMachine(); err != nil { + return nil + } + return tc.expect(machine) + } + gs.Eventually(checkExpectation, timeout).Should(Succeed()) + + // Check that resource version doesn't change if we call patchMachine() again + machineResourceVersion := machine.ResourceVersion + + gs.Expect(actuator.patchMachine(ctx, machine, machineToBePatched)).To(Succeed()) + gs.Eventually(getMachine, timeout).Should(Succeed()) + gs.Expect(machine.ResourceVersion).To(Equal(machineResourceVersion)) + }) + } +} + +func initActuator(mgr manager.Manager, t *testing.T) (*Actuator, error) { + codec, err := providerconfigv1.NewCodec() + if err != nil { + return nil, fmt.Errorf("unable to create codec: %v", err) + } + + mockCtrl := gomock.NewController(t) + mockAWSClient := mockaws.NewMockClient(mockCtrl) + + mockAWSClient.EXPECT().RunInstances(gomock.Any()).Return(stubReservation("ami-a9acbbd6", "i-02fcb933c5da7085c"), nil).AnyTimes() + mockAWSClient.EXPECT().DescribeInstances(gomock.Any()).Return(stubDescribeInstancesOutput("ami-a9acbbd6", "i-02fcb933c5da7085c", ec2.InstanceStateNameRunning), nil).AnyTimes() + mockAWSClient.EXPECT().TerminateInstances(gomock.Any()).Return(&ec2.TerminateInstancesOutput{}, nil).AnyTimes() + mockAWSClient.EXPECT().RegisterInstancesWithLoadBalancer(gomock.Any()).Return(nil, nil).AnyTimes() + mockAWSClient.EXPECT().ELBv2DescribeLoadBalancers(gomock.Any()).Return(stubDescribeLoadBalancersOutput(), nil).AnyTimes() + mockAWSClient.EXPECT().ELBv2DescribeTargetGroups(gomock.Any()).Return(stubDescribeTargetGroupsOutput(), nil).AnyTimes() + mockAWSClient.EXPECT().ELBv2RegisterTargets(gomock.Any()).Return(nil, nil).AnyTimes() + + eventsChannel := make(chan string, 1) + + params := ActuatorParams{ + Client: mgr.GetClient(), + Config: mgr.GetConfig(), + AwsClientBuilder: func(client client.Client, secretName, namespace, region string) (awsclient.Client, error) { + return mockAWSClient, nil + }, + Codec: codec, + // use fake recorder and store an event into one item long buffer for subsequent check + EventRecorder: &record.FakeRecorder{ + Events: eventsChannel, + }, + } + + actuator, err := NewActuator(params) + if err != nil { + return nil, fmt.Errorf("could not create AWS machine actuator: %v", err) + } + + return actuator, nil +} + +func getMachineStatus(codec *providerconfigv1.AWSProviderConfigCodec, machine *machinev1.Machine) (*providerconfigv1.AWSMachineProviderStatus, error) { + machineStatus := &providerconfigv1.AWSMachineProviderStatus{} + if err := codec.DecodeProviderStatus(machine.Status.ProviderStatus, machineStatus); err != nil { + return nil, fmt.Errorf("error decoding machine provider status: %v", err) + } + + return machineStatus, nil +}