diff --git a/cmd/kops/BUILD.bazel b/cmd/kops/BUILD.bazel index 063d7674d7e2b..18d6ce580cfc6 100644 --- a/cmd/kops/BUILD.bazel +++ b/cmd/kops/BUILD.bazel @@ -157,6 +157,7 @@ go_test( "//upup/pkg/fi/cloudup:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/cloudup/gce:go_default_library", + "//upup/pkg/fi/cloudup/openstack:go_default_library", "//util/pkg/ui:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", diff --git a/cmd/kops/lifecycle_integration_test.go b/cmd/kops/lifecycle_integration_test.go index bfee3307b4016..73f0047d737bb 100644 --- a/cmd/kops/lifecycle_integration_test.go +++ b/cmd/kops/lifecycle_integration_test.go @@ -19,6 +19,7 @@ package main import ( "bytes" "context" + "os" "path" "reflect" "sort" @@ -32,6 +33,7 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/upup/pkg/fi/cloudup/openstack" ) type LifecycleTestOptions struct { @@ -63,6 +65,14 @@ func TestLifecycleMinimal(t *testing.T) { }) } +func TestLifecycleMinimalOpenstack(t *testing.T) { + runLifecycleTestOpenstack(&LifecycleTestOptions{ + t: t, + SrcDir: "minimal_openstack", + ClusterName: "minimal-openstack.k8s.local", + }) +} + // TestLifecyclePrivateCalico runs the test on a private topology func TestLifecyclePrivateCalico(t *testing.T) { runLifecycleTestAWS(&LifecycleTestOptions{ @@ -122,7 +132,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio factory := util.NewFactory(factoryOptions) - beforeResources := AllResources(cloud) + beforeResources := AllAWSResources(cloud) { options := &CreateOptions{} @@ -187,7 +197,7 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio { var ids []string - for id := range AllResources(cloud) { + for id := range AllAWSResources(cloud) { ids = append(ids, id) } sort.Strings(ids) @@ -255,8 +265,8 @@ func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptio } } -// AllResources returns all resources -func AllResources(c *awsup.MockAWSCloud) map[string]interface{} { +// AllAWSResources returns all resources +func AllAWSResources(c *awsup.MockAWSCloud) map[string]interface{} { all := make(map[string]interface{}) for k, v := range c.MockEC2.(*mockec2.MockEC2).All() { all[k] = v @@ -264,6 +274,15 @@ func AllResources(c *awsup.MockAWSCloud) map[string]interface{} { return all } +// AllOpenstackResources returns all resources +func AllOpenstackResources(c *openstack.MockCloud) map[string]interface{} { + all := make(map[string]interface{}) + for k, v := range c.MockNovaClient.All() { + all[k] = v + } + return all +} + func runLifecycleTestAWS(o *LifecycleTestOptions) { o.AddDefaults() @@ -276,7 +295,7 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) { cloud := h.SetupMockAWS() var beforeIds []string - for id := range AllResources(cloud) { + for id := range AllAWSResources(cloud) { beforeIds = append(beforeIds, id) } sort.Strings(beforeIds) @@ -284,7 +303,7 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) { runLifecycleTest(h, o, cloud) var afterIds []string - for id := range AllResources(cloud) { + for id := range AllAWSResources(cloud) { afterIds = append(afterIds, id) } sort.Strings(afterIds) @@ -293,3 +312,122 @@ func runLifecycleTestAWS(o *LifecycleTestOptions) { t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds) } } + +func runLifecycleTestOpenstack(o *LifecycleTestOptions) { + o.AddDefaults() + + t := o.t + + h := testutils.NewIntegrationTestHarness(o.t) + defer h.Close() + + origRegion := os.Getenv("OS_REGION_NAME") + os.Setenv("OS_REGION_NAME", "us-test1") + defer func() { + os.Setenv("OS_REGION_NAME", origRegion) + }() + + h.MockKopsVersion("1.19.0-alpha.1") + cloud := h.SetupMockOpenstack() + + var beforeIds []string + for id := range AllOpenstackResources(cloud) { + beforeIds = append(beforeIds, id) + } + sort.Strings(beforeIds) + + ctx := context.Background() + + t.Logf("running lifecycle test for cluster %s", o.ClusterName) + + var stdout bytes.Buffer + + inputYAML := "in-" + o.Version + ".yaml" + + factoryOptions := &util.FactoryOptions{} + factoryOptions.RegistryPath = "memfs://tests" + + factory := util.NewFactory(factoryOptions) + + { + options := &CreateOptions{} + options.Filenames = []string{path.Join(o.SrcDir, inputYAML)} + + err := RunCreate(ctx, factory, &stdout, options) + if err != nil { + t.Fatalf("error running %q create: %v", inputYAML, err) + } + } + + { + options := &CreateSecretPublickeyOptions{} + options.ClusterName = o.ClusterName + options.Name = "admin" + options.PublicKeyPath = path.Join(o.SrcDir, "id_rsa.pub") + + err := RunCreateSecretPublicKey(ctx, factory, &stdout, options) + if err != nil { + t.Fatalf("error running %q create: %v", inputYAML, err) + } + } + + { + options := &UpdateClusterOptions{} + options.InitDefaults() + options.RunTasksOptions.MaxTaskDuration = 10 * time.Second + options.Yes = true + + // We don't test it here, and it adds a dependency on kubectl + options.CreateKubecfg = false + + _, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options) + if err != nil { + t.Fatalf("error running update cluster %q: %v", o.ClusterName, err) + } + } + + { + options := &UpdateClusterOptions{} + options.InitDefaults() + options.Target = cloudup.TargetDryRun + options.RunTasksOptions.MaxTaskDuration = 10 * time.Second + + // We don't test it here, and it adds a dependency on kubectl + options.CreateKubecfg = false + + results, err := RunUpdateCluster(ctx, factory, o.ClusterName, &stdout, options) + if err != nil { + t.Fatalf("error running update cluster %q: %v", o.ClusterName, err) + } + + target := results.Target.(*fi.DryRunTarget) + if target.HasChanges() { + var b bytes.Buffer + if err := target.PrintReport(results.TaskMap, &b); err != nil { + t.Fatalf("error building report: %v", err) + } + t.Fatalf("Target had changes after executing: %v", b.String()) + } + } + + { + options := &DeleteClusterOptions{} + options.Yes = true + options.ClusterName = o.ClusterName + if err := RunDeleteCluster(ctx, factory, &stdout, options); err != nil { + t.Fatalf("error running delete cluster %q: %v", o.ClusterName, err) + } + } + + { + var afterIds []string + for id := range AllOpenstackResources(cloud) { + afterIds = append(afterIds, id) + } + sort.Strings(afterIds) + + if !reflect.DeepEqual(beforeIds, afterIds) { + t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds) + } + } +} diff --git a/pkg/testutils/BUILD.bazel b/pkg/testutils/BUILD.bazel index f894db6da8f6c..8c1b275515745 100644 --- a/pkg/testutils/BUILD.bazel +++ b/pkg/testutils/BUILD.bazel @@ -16,6 +16,11 @@ go_library( "//cloudmock/aws/mockelbv2:go_default_library", "//cloudmock/aws/mockiam:go_default_library", "//cloudmock/aws/mockroute53:go_default_library", + "//cloudmock/openstack/mockblockstorage:go_default_library", + "//cloudmock/openstack/mockcompute:go_default_library", + "//cloudmock/openstack/mockdns:go_default_library", + "//cloudmock/openstack/mockloadbalancer:go_default_library", + "//cloudmock/openstack/mocknetworking:go_default_library", "//pkg/apis/kops:go_default_library", "//pkg/apis/kops/v1alpha2:go_default_library", "//pkg/kopscodecs:go_default_library", @@ -24,11 +29,18 @@ go_library( "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awsup:go_default_library", "//upup/pkg/fi/cloudup/gce:go_default_library", + "//upup/pkg/fi/cloudup/openstack:go_default_library", "//util/pkg/text:go_default_library", "//util/pkg/vfs:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/flavors:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/dns/v2/zones:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/imageservice/v2/images:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/networks:go_default_library", + "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/pkg/testutils/integrationtestharness.go b/pkg/testutils/integrationtestharness.go index 55c65d16a93e8..28d3aae4fa160 100644 --- a/pkg/testutils/integrationtestharness.go +++ b/pkg/testutils/integrationtestharness.go @@ -26,6 +26,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/route53" + "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" + "github.com/gophercloud/gophercloud/openstack/dns/v2/zones" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" "k8s.io/klog" kopsroot "k8s.io/kops" "k8s.io/kops/cloudmock/aws/mockautoscaling" @@ -34,10 +40,17 @@ import ( "k8s.io/kops/cloudmock/aws/mockelbv2" "k8s.io/kops/cloudmock/aws/mockiam" "k8s.io/kops/cloudmock/aws/mockroute53" + "k8s.io/kops/cloudmock/openstack/mockblockstorage" + "k8s.io/kops/cloudmock/openstack/mockcompute" + "k8s.io/kops/cloudmock/openstack/mockdns" + "k8s.io/kops/cloudmock/openstack/mockloadbalancer" + "k8s.io/kops/cloudmock/openstack/mocknetworking" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/pki" + "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/gce" + "k8s.io/kops/upup/pkg/fi/cloudup/openstack" "k8s.io/kops/util/pkg/vfs" ) @@ -239,6 +252,62 @@ func (h *IntegrationTestHarness) SetupMockGCE() { gce.InstallMockGCECloud("us-test1", "testproject") } +func (h *IntegrationTestHarness) SetupMockOpenstack() *openstack.MockCloud { + c := openstack.InstallMockOpenstackCloud("us-test1") + c.MockCinderClient = mockblockstorage.CreateClient() + + c.MockNeutronClient = mocknetworking.CreateClient() + + c.MockLBClient = mockloadbalancer.CreateClient() + + c.MockNovaClient = mockcompute.CreateClient() + + c.MockDNSClient = mockdns.CreateClient() + + extNetworkName := "external" + networkCreateOpts := networks.CreateOpts{ + Name: extNetworkName, + AdminStateUp: fi.Bool(true), + } + extNetwork := external.CreateOptsExt{ + CreateOptsBuilder: networkCreateOpts, + External: fi.Bool(true), + } + c.CreateNetwork(extNetwork) + c.SetExternalNetwork(&extNetworkName) + + extSubnetName := "external" + extSubnet := subnets.CreateOpts{ + Name: extSubnetName, + NetworkID: extNetworkName, + EnableDHCP: fi.Bool(true), + CIDR: "172.20.0.0/22", + } + c.CreateSubnet(extSubnet) + c.SetExternalSubnet(fi.String(extSubnetName)) + c.SetLBFloatingSubnet(fi.String(extSubnetName)) + images.Create(c.MockNovaClient.ServiceClient(), images.CreateOpts{ + Name: "Ubuntu-20.04", + MinDisk: 12, + }) + flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{ + Name: "n1-standard-2", + RAM: 8192, + VCPUs: 4, + Disk: fi.Int(16), + }) + flavors.Create(c.MockNovaClient.ServiceClient(), flavors.CreateOpts{ + Name: "n1-standard-1", + RAM: 8192, + VCPUs: 4, + Disk: fi.Int(16), + }) + zones.Create(c.MockDNSClient.ServiceClient(), zones.CreateOpts{ + Name: "minimal-openstack.k8s.local", + }) + return c +} + // MockKopsVersion will set the kops version to the specified value, until Close is called func (h *IntegrationTestHarness) MockKopsVersion(version string) { if h.originalKopsVersion != "" { diff --git a/tests/integration/update_cluster/minimal_openstack/id_rsa.pub b/tests/integration/update_cluster/minimal_openstack/id_rsa.pub new file mode 100755 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/minimal_openstack/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/minimal_openstack/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_openstack/in-v1alpha2.yaml new file mode 100644 index 0000000000000..60f775587d852 --- /dev/null +++ b/tests/integration/update_cluster/minimal_openstack/in-v1alpha2.yaml @@ -0,0 +1,91 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal-openstack.k8s.local +spec: + api: + dns: {} + authorization: + alwaysAllow: {} + channel: stable + cloudConfig: + openstack: {} + cloudProvider: openstack + configBase: memfs://tests/minimal-openstack.k8s.local + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test1-a + name: "1" + volumeType: test + name: main + - etcdMembers: + - instanceGroup: master-us-test1-a + name: "1" + volumeType: test + name: events + openstackServiceAccount: default + iam: + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: v1.14.0 + masterPublicName: api.minimal-openstack.k8s.local + networking: + kubenet: {} + networkCIDR: 192.168.0.0/16 + nonMasqueradeCIDR: 100.64.0.0/10 + project: testproject + sshAccess: + - 0.0.0.0/0 + subnets: + - name: us-test1 + region: us-test1 + type: Public + topology: + dns: + type: Private + masters: private + nodes: private + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-openstack.k8s.local + name: master-us-test1-a +spec: + image: Ubuntu-20.04 + machineType: n1-standard-1 + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + zones: + - us-test1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-openstack.k8s.local + name: nodes +spec: + image: Ubuntu-20.04 + machineType: n1-standard-2 + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test1 + zones: + - us-test1-a