Skip to content

Commit

Permalink
Add openstack integration test.
Browse files Browse the repository at this point in the history 
This will create / update / update / delete an openstack cluster using cloudmock, ensuring there are no lingering changes reported or orphaned resources
  • Loading branch information
@rifelpet
rifelpet committed Aug 10, 2020
1 parent a852a9d commit 6991655
Show file tree
Hide file tree
Showing 6 changed files with 318 additions and 6 deletions.
1 change: 1 addition & 0 deletions cmd/kops/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ go_test(
"//upup/pkg/fi/cloudup:go_default_library",
"//upup/pkg/fi/cloudup/awsup:go_default_library",
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//upup/pkg/fi/cloudup/openstack:go_default_library",
"//util/pkg/ui:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
Expand Down
150 changes: 144 additions & 6 deletions cmd/kops/lifecycle_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package main
import (
"bytes"
"context"
"os"
"path"
"reflect"
"sort"
Expand All @@ -32,6 +33,7 @@ import (
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
)

type LifecycleTestOptions struct {
Expand Down Expand Up @@ -63,6 +65,14 @@ func TestLifecycleMinimal(t *testing.T) {
})
}

func TestLifecycleMinimalOpenstack(t *testing.T) {
runLifecycleTestOpenstack(&LifecycleTestOptions{
t: t,
SrcDir: "minimal_openstack",
ClusterName: "minimal-openstack.k8s.local",
})
}

// TestLifecyclePrivateCalico runs the test on a private topology
func TestLifecyclePrivateCalico(t *testing.T) {
runLifecycleTestAWS(&LifecycleTestOptions{
Expand Down Expand Up @@ -122,7 +132,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio

factory := util.NewFactory(factoryOptions)

beforeResources := AllResources(cloud)
beforeResources := AllAWSResources(cloud)

{
options := &CreateOptions{}
Expand Down Expand Up @@ -187,7 +197,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio

{
var ids []string
for id := range AllResources(cloud) {
for id := range AllAWSResources(cloud) {
ids = append(ids, id)
}
sort.Strings(ids)
Expand Down Expand Up @@ -255,15 +265,24 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio
}
}

// AllResources returns all resources
func AllResources(c *awsup.MockAWSCloud) map[string]interface{} {
// AllAWSResources returns all resources
func AllAWSResources(c *awsup.MockAWSCloud) map[string]interface{} {
all := make(map[string]interface{})
for k, v := range c.MockEC2.(*mockec2.MockEC2).All() {
all[k] = v
}
return all
}

// AllOpenstackResources returns all resources
func AllOpenstackResources(c *openstack.MockCloud) map[string]interface{} {
all := make(map[string]interface{})
for k, v := range c.MockNovaClient.All() {
all[k] = v
}
return all
}

func runLifecycleTestAWS(o *LifecycleTestOptions) {
o.AddDefaults()

Expand All @@ -276,15 +295,15 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
cloud := h.SetupMockAWS()

var beforeIds []string
for id := range AllResources(cloud) {
for id := range AllAWSResources(cloud) {
beforeIds = append(beforeIds, id)
}
sort.Strings(beforeIds)

runLifecycleTest(h, o, cloud)

var afterIds []string
for id := range AllResources(cloud) {
for id := range AllAWSResources(cloud) {
afterIds = append(afterIds, id)
}
sort.Strings(afterIds)
Expand All @@ -293,3 +312,122 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) {
t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
}
}

func runLifecycleTestOpenstack(o *LifecycleTestOptions) {
o.AddDefaults()

t := o.t

h := testutils.NewIntegrationTestHarness(o.t)
defer h.Close()

origRegion := os.Getenv("OS_REGION_NAME")
os.Setenv("OS_REGION_NAME", "us-test1")
defer func() {
os.Setenv("OS_REGION_NAME", origRegion)
}()

h.MockKopsVersion("1.19.0-alpha.1")
cloud := h.SetupMockOpenstack()

var beforeIds []string
for id := range AllOpenstackResources(cloud) {
beforeIds = append(beforeIds, id)
}
sort.Strings(beforeIds)

ctx := context.Background()

t.Logf("running lifecycle test for cluster %s", o.ClusterName)

var stdout bytes.Buffer

inputYAML := "in-" + o.Version + ".yaml"

factoryOptions := &util.FactoryOptions{}
factoryOptions.RegistryPath = "memfs://tests"

factory := util.NewFactory(factoryOptions)

{
options := &CreateOptions{}
options.Filenames = []string{path.Join(o.SrcDir, inputYAML)}

err := RunCreate(ctx, factory, &stdout, options)
if err != nil {
t.Fatalf("error running %q create: %v", inputYAML, err)
}
}

{
options := &CreateSecretPublickeyOptions{}
options.ClusterName = o.ClusterName
options.Name = "admin"
options.PublicKeyPath = path.Join(o.SrcDir, "id_rsa.pub")

err := RunCreateSecretPublicKey(ctx, factory, &stdout, options)
if err != nil {
t.Fatalf("error running %q create: %v", inputYAML, err)
}
}

{
options := &UpdateClusterOptions{}
options.InitDefaults()
options.RunTasksOptions.MaxTaskDuration = 10 * time.Second
options.Yes = true

// We don't test it here, and it adds a dependency on kubectl
options.CreateKubecfg = false

_, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options)
if err != nil {
t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
}
}

{
options := &UpdateClusterOptions{}
options.InitDefaults()
options.Target = cloudup.TargetDryRun
options.RunTasksOptions.MaxTaskDuration = 10 * time.Second

// We don't test it here, and it adds a dependency on kubectl
options.CreateKubecfg = false

results, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options)
if err != nil {
t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
}

target := results.Target.(*fi.DryRunTarget)
if target.HasChanges() {
var b bytes.Buffer
if err := target.PrintReport(results.TaskMap, &b); err != nil {
t.Fatalf("error building report: %v", err)
}
t.Fatalf("Target had changes after executing: %v", b.String())
}
}

{
options := &DeleteClusterOptions{}
options.Yes = true
options.ClusterName = o.ClusterName
if err := RunDeleteCluster(ctx, factory, &stdout, options); err != nil {
t.Fatalf("error running delete cluster %q: %v", o.ClusterName, err)
}
}

{
var afterIds []string
for id := range AllOpenstackResources(cloud) {
afterIds = append(afterIds, id)
}
sort.Strings(afterIds)

if !reflect.DeepEqual(beforeIds, afterIds) {
t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
}
}
}
12 changes: 12 additions & 0 deletions pkg/testutils/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ go_library(
"//cloudmock/aws/mockelbv2:go_default_library",
"//cloudmock/aws/mockiam:go_default_library",
"//cloudmock/aws/mockroute53:go_default_library",
"//cloudmock/openstack/mockblockstorage:go_default_library",
"//cloudmock/openstack/mockcompute:go_default_library",
"//cloudmock/openstack/mockdns:go_default_library",
"//cloudmock/openstack/mockloadbalancer:go_default_library",
"//cloudmock/openstack/mocknetworking:go_default_library",
"//pkg/apis/kops:go_default_library",
"//pkg/apis/kops/v1alpha2:go_default_library",
"//pkg/kopscodecs:go_default_library",
Expand All @@ -24,11 +29,18 @@ go_library(
"//upup/pkg/fi:go_default_library",
"//upup/pkg/fi/cloudup/awsup:go_default_library",
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//upup/pkg/fi/cloudup/openstack:go_default_library",
"//util/pkg/text:go_default_library",
"//util/pkg/vfs:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/flavors:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/images:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/networks:go_default_library",
"//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
],
Expand Down
69 changes: 69 additions & 0 deletions pkg/testutils/integrationtestharness.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
"github.com/gophercloud/gophercloud/openstack/dns/v2/zones"
"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external"
"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
"k8s.io/klog"
kopsroot "k8s.io/kops"
"k8s.io/kops/cloudmock/aws/mockautoscaling"
Expand All @@ -34,10 +40,17 @@ import (
"k8s.io/kops/cloudmock/aws/mockelbv2"
"k8s.io/kops/cloudmock/aws/mockiam"
"k8s.io/kops/cloudmock/aws/mockroute53"
"k8s.io/kops/cloudmock/openstack/mockblockstorage"
"k8s.io/kops/cloudmock/openstack/mockcompute"
"k8s.io/kops/cloudmock/openstack/mockdns"
"k8s.io/kops/cloudmock/openstack/mockloadbalancer"
"k8s.io/kops/cloudmock/openstack/mocknetworking"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/pki"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/util/pkg/vfs"
)

Expand Down Expand Up @@ -239,6 +252,62 @@ func (h *IntegrationTestHarness) SetupMockGCE() {
gce.InstallMockGCECloud("us-test1", "testproject")
}

func (h *IntegrationTestHarness) SetupMockOpenstack() *openstack.MockCloud {
c := openstack.InstallMockOpenstackCloud("us-test1")
c.MockCinderClient = mockblockstorage.CreateClient()

c.MockNeutronClient = mocknetworking.CreateClient()

c.MockLBClient = mockloadbalancer.CreateClient()

c.MockNovaClient = mockcompute.CreateClient()

c.MockDNSClient = mockdns.CreateClient()

extNetworkName := "external"
networkCreateOpts := networks.CreateOpts{
Name: extNetworkName,
AdminStateUp: fi.Bool(true),
}
extNetwork := external.CreateOptsExt{
CreateOptsBuilder: networkCreateOpts,
External: fi.Bool(true),
}
c.CreateNetwork(extNetwork)
c.SetExternalNetwork(&extNetworkName)

extSubnetName := "external"
extSubnet := subnets.CreateOpts{
Name: extSubnetName,
NetworkID: extNetworkName,
EnableDHCP: fi.Bool(true),
CIDR: "172.20.0.0/22",
}
c.CreateSubnet(extSubnet)
c.SetExternalSubnet(fi.String(extSubnetName))
c.SetLBFloatingSubnet(fi.String(extSubnetName))
images.Create(c.MockNovaClient.ServiceClient(), images.CreateOpts{
Name: "Ubuntu-20.04",
MinDisk: 12,
})
flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{
Name: "n1-standard-2",
RAM: 8192,
VCPUs: 4,
Disk: fi.Int(16),
})
flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{
Name: "n1-standard-1",
RAM: 8192,
VCPUs: 4,
Disk: fi.Int(16),
})
zones.Create(c.MockDNSClient.ServiceClient(), zones.CreateOpts{
Name: "minimal-openstack.k8s.local",
})
return c
}

// MockKopsVersion will set the kops version to the specified value, until Close is called
func (h *IntegrationTestHarness) MockKopsVersion(version string) {
if h.originalKopsVersion != "" {
Expand Down
1 change: 1 addition & 0 deletions tests/integration/update_cluster/minimal_openstack/id_rsa.pub
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==
Loading

0 comments on commit 6991655

Please sign in to comment.