From 6308420635f1c6f831e7c9d942d15161f13f2e7f Mon Sep 17 00:00:00 2001 From: xing-yang Date: Sat, 19 Oct 2019 14:48:38 +0000 Subject: [PATCH 1/2] Split snapshot controller using beta APIs --- Makefile | 4 +- Dockerfile => cmd/csi-snapshotter/Dockerfile | 2 +- cmd/csi-snapshotter/main.go | 48 +- cmd/snapshot-controller/Dockerfile | 6 + cmd/snapshot-controller/main.go | 159 +++ cmd/snapshot-controller/main_test.go | 161 +++ .../{ => csi-snapshotter}/README.md | 0 .../rbac-csi-snapshotter.yaml} | 18 - .../rbac-external-provisioner.yaml | 0 .../setup-csi-snapshotter.yaml | 16 +- .../rbac-snapshot-controller.yaml | 80 ++ .../setup-snapshot-controller.yaml | 27 + examples/kubernetes/snapshot.yaml | 7 +- examples/kubernetes/snapshotclass.yaml | 5 +- examples/kubernetes/storageclass.yaml | 2 +- .../framework_test.go | 146 +- pkg/common-controller/snapshot_controller.go | 1190 ++++++++++++++++ .../snapshot_controller_base.go | 101 +- .../snapshot_controller_test.go | 11 +- .../snapshot_create_test.go | 173 +-- .../snapshot_delete_test.go | 185 +-- .../snapshot_finalizer_test.go | 14 +- .../snapshot_update_test.go} | 162 ++- pkg/controller/snapshot_controller.go | 1226 ----------------- .../csi_handler.go | 49 +- pkg/sidecar-controller/snapshot_controller.go | 539 ++++++++ .../snapshot_controller_base.go | 280 ++++ pkg/snapshotter/snapshotter.go | 13 +- pkg/snapshotter/snapshotter_test.go | 49 +- pkg/{controller => utils}/util.go | 128 +- pkg/{controller => utils}/util_test.go | 58 +- vendor/modules.txt | 4 +- 32 files changed, 3048 insertions(+), 1815 deletions(-) rename Dockerfile => cmd/csi-snapshotter/Dockerfile (74%) create mode 100644 cmd/snapshot-controller/Dockerfile create mode 100644 cmd/snapshot-controller/main.go create mode 100644 cmd/snapshot-controller/main_test.go rename deploy/kubernetes/{ => csi-snapshotter}/README.md (100%) rename deploy/kubernetes/{rbac.yaml => csi-snapshotter/rbac-csi-snapshotter.yaml} (79%) rename deploy/kubernetes/{ => csi-snapshotter}/rbac-external-provisioner.yaml (100%) rename deploy/kubernetes/{ => csi-snapshotter}/setup-csi-snapshotter.yaml (86%) create mode 100644 deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml create mode 100644 deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml rename pkg/{controller => common-controller}/framework_test.go (92%) create mode 100644 pkg/common-controller/snapshot_controller.go rename pkg/{controller => common-controller}/snapshot_controller_base.go (80%) rename pkg/{controller => common-controller}/snapshot_controller_test.go (93%) rename pkg/{controller => common-controller}/snapshot_create_test.go (63%) rename pkg/{controller => common-controller}/snapshot_delete_test.go (64%) rename pkg/{controller => common-controller}/snapshot_finalizer_test.go (85%) rename pkg/{controller/snapshot_ready_test.go => common-controller/snapshot_update_test.go} (68%) delete mode 100644 pkg/controller/snapshot_controller.go rename pkg/{controller => sidecar-controller}/csi_handler.go (62%) create mode 100644 pkg/sidecar-controller/snapshot_controller.go create mode 100644 pkg/sidecar-controller/snapshot_controller_base.go rename pkg/{controller => utils}/util.go (73%) rename pkg/{controller => utils}/util_test.go (85%) diff --git a/Makefile b/Makefile index 4a81c5107..4fca4dde4 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: all csi-snapshotter clean test +.PHONY: all snapshot-controller csi-snapshotter clean test -CMDS=csi-snapshotter +CMDS=snapshot-controller csi-snapshotter all: build include release-tools/build.make diff --git a/Dockerfile b/cmd/csi-snapshotter/Dockerfile similarity index 74% rename from Dockerfile rename to cmd/csi-snapshotter/Dockerfile index bf355958f..f52219564 100644 --- a/Dockerfile +++ b/cmd/csi-snapshotter/Dockerfile @@ -1,6 +1,6 @@ FROM gcr.io/distroless/static:latest LABEL maintainers="Kubernetes Authors" -LABEL description="CSI External Snapshotter" +LABEL description="CSI External Snapshotter Sidecar" COPY ./bin/csi-snapshotter csi-snapshotter ENTRYPOINT ["/csi-snapshotter"] diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 9d7c4ef5a..32ebb4125 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" - "github.com/kubernetes-csi/external-snapshotter/pkg/controller" + controller "github.com/kubernetes-csi/external-snapshotter/pkg/sidecar-controller" "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" @@ -56,17 +56,13 @@ const ( // Command line flags var ( - snapshotterName = flag.String("snapshotter", "", "This option is deprecated.") - kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") - connectionTimeout = flag.Duration("connection-timeout", 0, "The --connection-timeout flag is deprecated") - csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.") - createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.") - createSnapshotContentInterval = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.") - resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") - snapshotNamePrefix = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot") - snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.") - showVersion = flag.Bool("version", false, "Show version.") - csiTimeout = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.") + kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") + csiAddress = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.") + resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") + snapshotNamePrefix = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot") + snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.") + showVersion = flag.Bool("version", false, "Show version.") + csiTimeout = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.") leaderElection = flag.Bool("leader-election", false, "Enables leader election.") leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.") @@ -88,14 +84,6 @@ func main() { } klog.Infof("Version: %s", version) - if *connectionTimeout != 0 { - klog.Warning("--connection-timeout is deprecated and will have no effect") - } - - if *snapshotterName != "" { - klog.Warning("--snapshotter is deprecated and will have no effect") - } - // Create the client config. Use kubeconfig if given, otherwise assume in-cluster. config, err := buildConfig(*kubeconfig) if err != nil { @@ -133,13 +121,13 @@ func main() { defer cancel() // Find driver name - *snapshotterName, err = csirpc.GetDriverName(ctx, csiConn) + driverName, err := csirpc.GetDriverName(ctx, csiConn) if err != nil { klog.Errorf("error getting CSI driver name: %v", err) os.Exit(1) } - klog.V(2).Infof("CSI driver name: %q", *snapshotterName) + klog.V(2).Infof("CSI driver name: %q", driverName) // Check it's ready if err = csirpc.ProbeForever(csiConn, *csiTimeout); err != nil { @@ -154,7 +142,7 @@ func main() { os.Exit(1) } if !supportsCreateSnapshot { - klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", *snapshotterName) + klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", driverName) os.Exit(1) } @@ -163,19 +151,15 @@ func main() { os.Exit(1) } - klog.V(2).Infof("Start NewCSISnapshotController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%+v] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", *snapshotterName, *kubeconfig, *csiTimeout, *csiAddress, createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength) + klog.V(2).Infof("Start NewCSISnapshotSideCarController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", driverName, *kubeconfig, *csiTimeout, *csiAddress, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength) snapShotter := snapshotter.NewSnapshotter(csiConn) - ctrl := controller.NewCSISnapshotController( + ctrl := controller.NewCSISnapshotSideCarController( snapClient, kubeClient, - *snapshotterName, - factory.Snapshot().V1beta1().VolumeSnapshots(), + driverName, factory.Snapshot().V1beta1().VolumeSnapshotContents(), factory.Snapshot().V1beta1().VolumeSnapshotClasses(), - coreFactory.Core().V1().PersistentVolumeClaims(), - *createSnapshotContentRetryCount, - *createSnapshotContentInterval, snapShotter, *csiTimeout, *resyncPeriod, @@ -200,7 +184,7 @@ func main() { if !*leaderElection { run(context.TODO()) } else { - lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(*snapshotterName, "/", "-", -1)) + lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(driverName, "/", "-", -1)) le := leaderelection.NewLeaderElection(kubeClient, lockName, run) if *leaderElectionNamespace != "" { le.WithNamespace(*leaderElectionNamespace) diff --git a/cmd/snapshot-controller/Dockerfile b/cmd/snapshot-controller/Dockerfile new file mode 100644 index 000000000..b29db098e --- /dev/null +++ b/cmd/snapshot-controller/Dockerfile @@ -0,0 +1,6 @@ +FROM gcr.io/distroless/static:latest +LABEL maintainers="Kubernetes Authors" +LABEL description="Snapshot Controller" + +COPY ./bin/snapshot-controller snapshot-controller +ENTRYPOINT ["/snapshot-controller"] diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go new file mode 100644 index 000000000..9895f350f --- /dev/null +++ b/cmd/snapshot-controller/main.go @@ -0,0 +1,159 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "time" + + "google.golang.org/grpc" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/csi-lib-utils/leaderelection" + csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" + controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller" + + clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" + informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" + coreinformers "k8s.io/client-go/informers" +) + +const ( + // Number of worker threads + threads = 10 +) + +// Command line flags +var ( + kubeconfig = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") + createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.") + createSnapshotContentInterval = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.") + resyncPeriod = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.") + showVersion = flag.Bool("version", false, "Show version.") + + leaderElection = flag.Bool("leader-election", false, "Enables leader election.") + leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.") +) + +var ( + version = "unknown" +) + +func main() { + klog.InitFlags(nil) + flag.Set("logtostderr", "true") + flag.Parse() + + if *showVersion { + fmt.Println(os.Args[0], version) + os.Exit(0) + } + klog.Infof("Version: %s", version) + + // Create the client config. Use kubeconfig if given, otherwise assume in-cluster. + config, err := buildConfig(*kubeconfig) + if err != nil { + klog.Error(err.Error()) + os.Exit(1) + } + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Error(err.Error()) + os.Exit(1) + } + + snapClient, err := clientset.NewForConfig(config) + if err != nil { + klog.Errorf("Error building snapshot clientset: %s", err.Error()) + os.Exit(1) + } + + factory := informers.NewSharedInformerFactory(snapClient, *resyncPeriod) + coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, *resyncPeriod) + + // Add Snapshot types to the defualt Kubernetes so events can be logged for them + snapshotscheme.AddToScheme(scheme.Scheme) + + klog.V(2).Infof("Start NewCSISnapshotController with kubeconfig [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%d] resyncPeriod [%+v]", *kubeconfig, *createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod) + + ctrl := controller.NewCSISnapshotCommonController( + snapClient, + kubeClient, + factory.Snapshot().V1beta1().VolumeSnapshots(), + factory.Snapshot().V1beta1().VolumeSnapshotContents(), + factory.Snapshot().V1beta1().VolumeSnapshotClasses(), + coreFactory.Core().V1().PersistentVolumeClaims(), + *createSnapshotContentRetryCount, + *createSnapshotContentInterval, + *resyncPeriod, + ) + + run := func(context.Context) { + // run... + stopCh := make(chan struct{}) + factory.Start(stopCh) + coreFactory.Start(stopCh) + go ctrl.Run(threads, stopCh) + + // ...until SIGINT + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + close(stopCh) + } + + if !*leaderElection { + run(context.TODO()) + } else { + lockName := "snapshot-controller-leader" + le := leaderelection.NewLeaderElection(kubeClient, lockName, run) + if *leaderElectionNamespace != "" { + le.WithNamespace(*leaderElectionNamespace) + } + if err := le.Run(); err != nil { + klog.Fatalf("failed to initialize leader election: %v", err) + } + } +} + +func buildConfig(kubeconfig string) (*rest.Config, error) { + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} + +func supportsControllerCreateSnapshot(ctx context.Context, conn *grpc.ClientConn) (bool, error) { + capabilities, err := csirpc.GetControllerCapabilities(ctx, conn) + if err != nil { + return false, err + } + + return capabilities[csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT], nil +} diff --git a/cmd/snapshot-controller/main_test.go b/cmd/snapshot-controller/main_test.go new file mode 100644 index 000000000..f13aba72b --- /dev/null +++ b/cmd/snapshot-controller/main_test.go @@ -0,0 +1,161 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/mock/gomock" + "github.com/kubernetes-csi/csi-lib-utils/connection" + "github.com/kubernetes-csi/csi-test/driver" + + "google.golang.org/grpc" +) + +func Test_supportsControllerCreateSnapshot(t *testing.T) { + tests := []struct { + name string + output *csi.ControllerGetCapabilitiesResponse + injectError bool + expectError bool + expectResult bool + }{ + { + name: "success", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, + }, + }, + expectError: false, + expectResult: true, + }, + { + name: "gRPC error", + output: nil, + injectError: true, + expectError: true, + expectResult: false, + }, + { + name: "no create snapshot", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + }, + }, + expectError: false, + expectResult: false, + }, + { + name: "empty capability", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: nil, + }, + }, + }, + expectError: false, + expectResult: false, + }, + { + name: "no capabilities", + output: &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{}, + }, + expectError: false, + expectResult: false, + }, + } + + mockController, driver, _, controllerServer, csiConn, err := createMockServer(t) + if err != nil { + t.Fatal(err) + } + defer mockController.Finish() + defer driver.Stop() + defer csiConn.Close() + + for _, test := range tests { + + in := &csi.ControllerGetCapabilitiesRequest{} + + out := test.output + var injectedErr error + if test.injectError { + injectedErr = fmt.Errorf("mock error") + } + + // Setup expectation + controllerServer.EXPECT().ControllerGetCapabilities(gomock.Any(), in).Return(out, injectedErr).Times(1) + + ok, err := supportsControllerCreateSnapshot(context.Background(), csiConn) + if test.expectError && err == nil { + t.Errorf("test %q: Expected error, got none", test.name) + } + if !test.expectError && err != nil { + t.Errorf("test %q: got error: %v", test.name, err) + } + if err == nil && test.expectResult != ok { + t.Errorf("test fail expected result %t but got %t\n", test.expectResult, ok) + } + } +} + +func createMockServer(t *testing.T) (*gomock.Controller, *driver.MockCSIDriver, *driver.MockIdentityServer, *driver.MockControllerServer, *grpc.ClientConn, error) { + // Start the mock server + mockController := gomock.NewController(t) + identityServer := driver.NewMockIdentityServer(mockController) + controllerServer := driver.NewMockControllerServer(mockController) + drv := driver.NewMockCSIDriver(&driver.MockCSIDriverServers{ + Identity: identityServer, + Controller: controllerServer, + }) + drv.Start() + + // Create a client connection to it + addr := drv.Address() + csiConn, err := connection.Connect(addr) + if err != nil { + return nil, nil, nil, nil, nil, err + } + + return mockController, drv, identityServer, controllerServer, csiConn, nil +} diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/csi-snapshotter/README.md similarity index 100% rename from deploy/kubernetes/README.md rename to deploy/kubernetes/csi-snapshotter/README.md diff --git a/deploy/kubernetes/rbac.yaml b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml similarity index 79% rename from deploy/kubernetes/rbac.yaml rename to deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml index 5bed69e61..9ca4e00a7 100644 --- a/deploy/kubernetes/rbac.yaml +++ b/deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml @@ -20,15 +20,6 @@ metadata: # rename if there are conflicts name: external-snapshotter-runner rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] @@ -48,15 +39,6 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents/status"] verbs: ["update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update"] --- kind: ClusterRoleBinding diff --git a/deploy/kubernetes/rbac-external-provisioner.yaml b/deploy/kubernetes/csi-snapshotter/rbac-external-provisioner.yaml similarity index 100% rename from deploy/kubernetes/rbac-external-provisioner.yaml rename to deploy/kubernetes/csi-snapshotter/rbac-external-provisioner.yaml diff --git a/deploy/kubernetes/setup-csi-snapshotter.yaml b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml similarity index 86% rename from deploy/kubernetes/setup-csi-snapshotter.yaml rename to deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml index eed194f29..3555b09f8 100644 --- a/deploy/kubernetes/setup-csi-snapshotter.yaml +++ b/deploy/kubernetes/csi-snapshotter/setup-csi-snapshotter.yaml @@ -1,6 +1,6 @@ # This YAML file shows how to deploy the CSI snapshotter together # with the hostpath CSI driver. It depends on the RBAC rules -# from rbac.yaml and rbac-external-provisioner.yaml. +# from rbac-csi-snapshotter.yaml and rbac-external-provisioner.yaml. # # Because external-snapshotter and external-provisioner get # deployed in the same pod, we have to merge the permissions @@ -72,11 +72,10 @@ spec: serviceAccount: csi-snapshotter containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:v1.3.0 + image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1 args: - - "--provisioner=csi-hostpath" + - "--v=5" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" env: - name: ADDRESS value: /csi/csi.sock @@ -85,20 +84,21 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: quay.io/k8scsi/csi-snapshotter:v1.2.0 + # NOTE: replace with official image when released: quay.io/k8scsi/csi-snapshotter:v2.0.0 + image: csi-snapshotter:testbeta #quay.io/k8scsi/csi-snapshotter:testbeta args: + - "--v=5" - "--csi-address=$(ADDRESS)" - - "--connection-timeout=15s" - "--leader-election=false" env: - name: ADDRESS value: /csi/csi.sock - imagePullPolicy: Always + imagePullPolicy: IfNotPresent #Always volumeMounts: - name: socket-dir mountPath: /csi - name: hostpath - image: quay.io/k8scsi/hostpathplugin:v1.1.0 + image: quay.io/k8scsi/hostpathplugin:v1.2.0 args: - "--v=5" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml new file mode 100644 index 000000000..72e0d157a --- /dev/null +++ b/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml @@ -0,0 +1,80 @@ +# RBAC file for the snapshot controller. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + # replace with non-default namespace name + namespace: default +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: default # TODO: replace with the namespace you want for your sidecar + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: default # TODO: replace with the namespace you want for your sidecar +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: default # TODO: replace with the namespace you want for your sidecar +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml new file mode 100644 index 000000000..853a35a42 --- /dev/null +++ b/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml @@ -0,0 +1,27 @@ +# This YAML file shows how to deploy the snapshot controller + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller +spec: + serviceName: "snapshot-controller" + replicas: 1 + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + # NOTE: replace with official image when released: quay.io/k8scsi/snapshot-controller:v2.0.0 + image: snapshot-controller:testbeta #quay.io/k8scsi/snapshot-controller:testbeta + args: + - "--v=5" + - "--leader-election=false" + imagePullPolicy: IfNotPresent #Always diff --git a/examples/kubernetes/snapshot.yaml b/examples/kubernetes/snapshot.yaml index b7a913f9c..86a102b88 100644 --- a/examples/kubernetes/snapshot.yaml +++ b/examples/kubernetes/snapshot.yaml @@ -1,9 +1,8 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 +apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshot metadata: name: new-snapshot-demo spec: - snapshotClassName: csi-hostpath-snapclass + volumeSnapshotClassName: csi-hostpath-snapclass source: - name: hpvc - kind: PersistentVolumeClaim + persistentVolumeClaimName: hpvc diff --git a/examples/kubernetes/snapshotclass.yaml b/examples/kubernetes/snapshotclass.yaml index dfa34df56..892dfd0c8 100644 --- a/examples/kubernetes/snapshotclass.yaml +++ b/examples/kubernetes/snapshotclass.yaml @@ -1,5 +1,6 @@ -apiVersion: snapshot.storage.k8s.io/v1alpha1 +apiVersion: snapshot.storage.k8s.io/v1beta1 kind: VolumeSnapshotClass metadata: name: csi-hostpath-snapclass -snapshotter: csi-hostpath +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/examples/kubernetes/storageclass.yaml b/examples/kubernetes/storageclass.yaml index c92797167..59999a8cc 100644 --- a/examples/kubernetes/storageclass.yaml +++ b/examples/kubernetes/storageclass.yaml @@ -2,6 +2,6 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-hostpath-sc -provisioner: csi-hostpath +provisioner: hostpath.csi.k8s.io #csi-hostpath reclaimPolicy: Delete volumeBindingMode: Immediate diff --git a/pkg/controller/framework_test.go b/pkg/common-controller/framework_test.go similarity index 92% rename from pkg/controller/framework_test.go rename to pkg/common-controller/framework_test.go index b2120843c..82531eae5 100644 --- a/pkg/controller/framework_test.go +++ b/pkg/common-controller/framework_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "context" @@ -35,6 +35,7 @@ import ( snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme" informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions" storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -115,7 +116,7 @@ type controllerTest struct { expectSuccess bool } -type testCall func(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error +type testCall func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error const testNamespace = "default" const mockDriverName = "csi-mock-plugin" @@ -152,7 +153,7 @@ type snapshotReactor struct { snapshots map[string]*crdv1.VolumeSnapshot changedObjects []interface{} changedSinceLastSync int - ctrl *csiSnapshotController + ctrl *csiSnapshotCommonController fakeContentWatch *watch.FakeWatcher fakeSnapshotWatch *watch.FakeWatcher lock sync.Mutex @@ -170,17 +171,19 @@ type reactorError struct { } func withSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshot { - snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer) + snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer) + snapshot.ObjectMeta.Finalizers = append(snapshot.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer) return snapshot } func withContentFinalizer(content *crdv1.VolumeSnapshotContent) *crdv1.VolumeSnapshotContent { - content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer) + content.ObjectMeta.Finalizers = append(content.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") return content } func withPVCFinalizer(pvc *v1.PersistentVolumeClaim) *v1.PersistentVolumeClaim { - pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, PVCFinalizer) + pvc.ObjectMeta.Finalizers = append(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer) return pvc } @@ -423,7 +426,9 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + } expectedMap[v.Name] = v } for _, v := range r.contents { @@ -432,7 +437,9 @@ func (r *snapshotReactor) checkContents(expectedContents []*crdv1.VolumeSnapshot v := v.DeepCopy() v.ResourceVersion = "" v.Spec.VolumeSnapshotRef.ResourceVersion = "" - v.Status.CreationTime = nil + if v.Status != nil { + v.Status.CreationTime = nil + } gotMap[v.Name] = v } if !reflect.DeepEqual(expectedMap, gotMap) { @@ -480,7 +487,7 @@ func (r *snapshotReactor) checkSnapshots(expectedSnapshots []*crdv1.VolumeSnapsh // checkEvents compares all expectedEvents with events generated during the test // and reports differences. -func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotController) error { +func checkEvents(t *testing.T, expectedEvents []string, ctrl *csiSnapshotCommonController) error { var err error // Read recorded events - wait up to 1 minute to get all the expected ones @@ -699,7 +706,7 @@ func (r *snapshotReactor) addSnapshotEvent(snapshot *crdv1.VolumeSnapshot) { } } -func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { +func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, ctrl *csiSnapshotCommonController, fakeVolumeWatch, fakeClaimWatch *watch.FakeWatcher, errors []reactorError) *snapshotReactor { reactor := &snapshotReactor{ secrets: make(map[string]*v1.Secret), storageClasses: make(map[string]*storagev1.StorageClass), @@ -732,36 +739,31 @@ func newSnapshotReactor(kubeClient *kubefake.Clientset, client *fake.Clientset, func alwaysReady() bool { return true } func newTestController(kubeClient kubernetes.Interface, clientset clientset.Interface, - informerFactory informers.SharedInformerFactory, t *testing.T, test controllerTest) (*csiSnapshotController, error) { + informerFactory informers.SharedInformerFactory, t *testing.T, test controllerTest) (*csiSnapshotCommonController, error) { if informerFactory == nil { - informerFactory = informers.NewSharedInformerFactory(clientset, NoResyncPeriodFunc()) + informerFactory = informers.NewSharedInformerFactory(clientset, utils.NoResyncPeriodFunc()) } - coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, NoResyncPeriodFunc()) + coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, utils.NoResyncPeriodFunc()) // Construct controller - fakeSnapshot := &fakeSnapshotter{ - t: t, - listCalls: test.expectedListCalls, - createCalls: test.expectedCreateCalls, - deleteCalls: test.expectedDeleteCalls, - } + //fakeSnapshot := &fakeSnapshotter{ + // t: t, + // listCalls: test.expectedListCalls, + // createCalls: test.expectedCreateCalls, + // deleteCalls: test.expectedDeleteCalls, + //} - ctrl := NewCSISnapshotController( + ctrl := NewCSISnapshotCommonController( clientset, kubeClient, - mockDriverName, informerFactory.Snapshot().V1beta1().VolumeSnapshots(), informerFactory.Snapshot().V1beta1().VolumeSnapshotContents(), informerFactory.Snapshot().V1beta1().VolumeSnapshotClasses(), coreFactory.Core().V1().PersistentVolumeClaims(), 3, 5*time.Millisecond, - fakeSnapshot, - 5*time.Millisecond, 60*time.Second, - "snapshot", - -1, ) ctrl.eventRecorder = record.NewFakeRecorder(1000) @@ -776,7 +778,8 @@ func newTestController(kubeClient kubernetes.Interface, clientset clientset.Inte func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, - withFinalizer bool) *crdv1.VolumeSnapshotContent { + withFinalizer bool, withStatus bool) *crdv1.VolumeSnapshotContent { + ready := true content := crdv1.VolumeSnapshotContent{ ObjectMeta: metav1.ObjectMeta{ Name: contentName, @@ -786,13 +789,17 @@ func newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHa Driver: mockDriverName, DeletionPolicy: deletionPolicy, }, - Status: &crdv1.VolumeSnapshotContentStatus{ + } + + if withStatus { + content.Status = &crdv1.VolumeSnapshotContentStatus{ CreationTime: creationTime, RestoreSize: size, - }, + ReadyToUse: &ready, + } } - if snapshotHandle != "" { + if withStatus && snapshotHandle != "" { content.Status.SnapshotHandle = &snapshotHandle } @@ -830,14 +837,22 @@ func newContentArray(contentName, boundToSnapshotUID, boundToSnapshotName, snaps deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { return []*crdv1.VolumeSnapshotContent{ - newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer), + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, true), + } +} + +func newContentArrayNoStatus(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, + deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, + withFinalizer bool, withStatus bool) []*crdv1.VolumeSnapshotContent { + return []*crdv1.VolumeSnapshotContent{ + newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, withStatus), } } func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, creationTime, size *int64, readyToUse *bool, withFinalizer bool) []*crdv1.VolumeSnapshotContent { - content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer) + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, creationTime, size, withFinalizer, true) content.Status.ReadyToUse = readyToUse return []*crdv1.VolumeSnapshotContent{ content, @@ -847,7 +862,7 @@ func newContentArrayWithReadyToUse(contentName, boundToSnapshotUID, boundToSnaps func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle string, deletionPolicy crdv1.DeletionPolicy, size, creationTime *int64, withFinalizer bool) []*crdv1.VolumeSnapshotContent { - content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer) + content := newContent(contentName, boundToSnapshotUID, boundToSnapshotName, snapshotHandle, snapshotClassName, desiredSnapshotHandle, volumeHandle, deletionPolicy, size, creationTime, withFinalizer, true) content.Spec.Driver = "fake" return []*crdv1.VolumeSnapshotContent{ content, @@ -857,7 +872,7 @@ func newContentWithUnmatchDriverArray(contentName, boundToSnapshotUID, boundToSn func newSnapshot( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError) *crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool) *crdv1.VolumeSnapshot { snapshot := crdv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: snapshotName, @@ -869,12 +884,15 @@ func newSnapshot( Spec: crdv1.VolumeSnapshotSpec{ VolumeSnapshotClassName: nil, }, - Status: &crdv1.VolumeSnapshotStatus{ + } + + if !nilStatus { + snapshot.Status = &crdv1.VolumeSnapshotStatus{ CreationTime: creationTime, ReadyToUse: readyToUse, Error: err, RestoreSize: restoreSize, - }, + } } if boundContentName != "" { @@ -898,9 +916,9 @@ func newSnapshot( func newSnapshotArray( snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName string, readyToUse *bool, creationTime *metav1.Time, restoreSize *resource.Quantity, - err *crdv1.VolumeSnapshotError) []*crdv1.VolumeSnapshot { + err *crdv1.VolumeSnapshotError, nilStatus bool) []*crdv1.VolumeSnapshot { return []*crdv1.VolumeSnapshot{ - newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err), + newSnapshot(snapshotName, snapshotUID, pvcName, targetContentName, snapshotClassName, boundContentName, readyToUse, creationTime, restoreSize, err, nilStatus), } } @@ -1015,11 +1033,11 @@ func newVolumeError(message string) *crdv1.VolumeSnapshotError { } } -func testSyncSnapshot(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncSnapshot(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { return ctrl.syncSnapshot(test.initialSnapshots[0]) } -func testSyncSnapshotError(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncSnapshotError(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { err := ctrl.syncSnapshot(test.initialSnapshots[0]) if err != nil { @@ -1028,16 +1046,16 @@ func testSyncSnapshotError(ctrl *csiSnapshotController, reactor *snapshotReactor return fmt.Errorf("syncSnapshot succeeded when failure was expected") } -func testSyncContent(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { +func testSyncContent(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { return ctrl.syncContent(test.initialContents[0]) } -func testAddPVCFinalizer(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { - return ctrl.ensureSnapshotSourceFinalizer(test.initialSnapshots[0]) +func testAddPVCFinalizer(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { + return ctrl.ensurePVCFinalizer(test.initialSnapshots[0]) } -func testRemovePVCFinalizer(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { - return ctrl.checkandRemoveSnapshotSourceFinalizer(test.initialSnapshots[0]) +func testRemovePVCFinalizer(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { + return ctrl.checkandRemovePVCFinalizer(test.initialSnapshots[0]) } var ( @@ -1062,9 +1080,9 @@ var ( // injected function to simulate that something is happening when the // controller waits for the operation lock. Controller is then resumed and we // check how it behaves. -func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(ctrl *csiSnapshotController, reactor *snapshotReactor)) testCall { +func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor)) testCall { - return func(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest) error { + return func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest) error { // Inject a hook before async operation starts klog.V(4).Infof("reactor:injecting call") injectBeforeOperation(ctrl, reactor) @@ -1089,7 +1107,7 @@ func wrapTestWithInjectedOperation(toWrap testCall, injectBeforeOperation func(c } } -func evaluateTestResults(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest, t *testing.T) { +func evaluateTestResults(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest, t *testing.T) { // Evaluate results if err := reactor.checkSnapshots(test.expectedSnapshots); err != nil { t.Errorf("Test %q: %v", test.name, err) @@ -1130,10 +1148,8 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 reactor.snapshots[snapshot.Name] = snapshot } for _, content := range test.initialContents { - if ctrl.isDriverMatch(test.initialContents[0]) { - ctrl.contentStore.Add(content) - reactor.contents[content.Name] = content - } + ctrl.contentStore.Add(content) + reactor.contents[content.Name] = content } pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) @@ -1162,7 +1178,7 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 // Run the tested functions err = test.test(ctrl, reactor, test) - if err != nil { + if test.expectSuccess && err != nil { t.Errorf("Test %q failed: %v", test.name, err) } @@ -1176,7 +1192,7 @@ func runSyncTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1 } } -// This tests ensureSnapshotSourceFinalizer and checkandRemoveSnapshotSourceFinalizer +// This tests ensurePVCFinalizer and checkandRemovePVCFinalizer func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses []*crdv1.VolumeSnapshotClass) { snapshotscheme.AddToScheme(scheme.Scheme) for _, test := range tests { @@ -1197,10 +1213,8 @@ func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses reactor.snapshots[snapshot.Name] = snapshot } for _, content := range test.initialContents { - if ctrl.isDriverMatch(test.initialContents[0]) { - ctrl.contentStore.Add(content) - reactor.contents[content.Name] = content - } + ctrl.contentStore.Add(content) + reactor.contents[content.Name] = content } pvcIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) @@ -1239,7 +1253,7 @@ func runPVCFinalizerTests(t *testing.T, tests []controllerTest, snapshotClasses } // Evaluate PVCFinalizer tests results -func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotReactor, test controllerTest, t *testing.T) { +func evaluatePVCFinalizerTests(ctrl *csiSnapshotCommonController, reactor *snapshotReactor, test controllerTest, t *testing.T) { // Evaluate results bHasPVCFinalizer := false name := sysruntime.FuncForPC(reflect.ValueOf(test.test).Pointer()).Name() @@ -1255,7 +1269,7 @@ func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotRea if funcName == "testAddPVCFinalizer" { for _, pvc := range reactor.claims { if test.initialClaims[0].Name == pvc.Name { - if !slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, PVCFinalizer, nil) && slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { + if !slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, utils.PVCFinalizer, nil) && slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { klog.V(4).Infof("test %q succeeded. PVCFinalizer is added to PVC %s", test.name, pvc.Name) bHasPVCFinalizer = true } @@ -1270,7 +1284,7 @@ func evaluatePVCFinalizerTests(ctrl *csiSnapshotController, reactor *snapshotRea if funcName == "testRemovePVCFinalizer" { for _, pvc := range reactor.claims { if test.initialClaims[0].Name == pvc.Name { - if slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, PVCFinalizer, nil) && !slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { + if slice.ContainsString(test.initialClaims[0].ObjectMeta.Finalizers, utils.PVCFinalizer, nil) && !slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { klog.V(4).Infof("test %q succeeded. PVCFinalizer is removed from PVC %s", test.name, pvc.Name) bHasPVCFinalizer = false } @@ -1310,23 +1324,23 @@ func secret() *v1.Secret { func secretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "secret", - AnnDeletionSecretRefNamespace: "default", + utils.AnnDeletionSecretRefName: "secret", + utils.AnnDeletionSecretRefNamespace: "default", } } func emptyNamespaceSecretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "name", - AnnDeletionSecretRefNamespace: "", + utils.AnnDeletionSecretRefName: "name", + utils.AnnDeletionSecretRefNamespace: "", } } // this refers to emptySecret(), which is missing data. func emptyDataSecretAnnotations() map[string]string { return map[string]string{ - AnnDeletionSecretRefName: "emptysecret", - AnnDeletionSecretRefNamespace: "default", + utils.AnnDeletionSecretRefName: "emptysecret", + utils.AnnDeletionSecretRefNamespace: "default", } } diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go new file mode 100644 index 000000000..ec5842df8 --- /dev/null +++ b/pkg/common-controller/snapshot_controller.go @@ -0,0 +1,1190 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common_controller + +import ( + "fmt" + "strings" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/scheme" + ref "k8s.io/client-go/tools/reference" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/slice" +) + +// ================================================================== +// PLEASE DO NOT ATTEMPT TO SIMPLIFY THIS CODE. +// KEEP THE SPACE SHUTTLE FLYING. +// ================================================================== + +// Design: +// +// The fundamental key to this design is the bi-directional "pointer" between +// VolumeSnapshots and VolumeSnapshotContents, which is represented here +// as snapshot.Status.BoundVolumeSnapshotContentName and content.Spec.VolumeSnapshotRef. +// The bi-directionality is complicated to manage in a transactionless system, but +// without it we can't ensure sane behavior in the face of different forms of +// trouble. For example, a rogue HA controller instance could end up racing +// and making multiple bindings that are indistinguishable, resulting in +// potential data loss. +// +// This controller is designed to work in active-passive high availability +// mode. It *could* work also in active-active HA mode, all the object +// transitions are designed to cope with this, however performance could be +// lower as these two active controllers will step on each other toes +// frequently. +// +// This controller supports both dynamic snapshot creation and pre-bound snapshot. +// In pre-bound mode, objects are created with pre-defined pointers: a VolumeSnapshot +// points to a specific VolumeSnapshotContent and the VolumeSnapshotContent also +// points back for this VolumeSnapshot. +// +// The snapshot controller is split into two controllers in its beta phase: a +// common controller that is deployed on the kubernetes master node and a sidecar +// controller that is deployed with the CSI driver. + +// The dynamic snapshot creation is multi-step process: first common controller +// creates snapshot content object, then the snapshot sidecar triggers snapshot +// creation though csi volume driver and updates snapshot content status with +// snapshotHandle, creationTime, restoreSize, readyToUse, and error fields. The +// common controller updates snapshot status based on content status until +// bi-directional binding is complete and readyToUse becomes true. Error field +// in the snapshot status will be updated accordingly when failure occurrs. + +const snapshotKind = "VolumeSnapshot" +const snapshotAPIGroup = crdv1.GroupName + +const controllerUpdateFailMsg = "snapshot controller failed to update" + +// syncContent deals with one key off the queue. It returns false when it's time to quit. +func (ctrl *csiSnapshotCommonController) syncContent(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) + + snapshotName := utils.SnapshotRefKey(&content.Spec.VolumeSnapshotRef) + + if utils.NeedToAddContentFinalizer(content) { + // Content is not being deleted -> it should have the finalizer. + klog.V(5).Infof("syncContent: Add Finalizer for VolumeSnapshotContent[%s]", content.Name) + return ctrl.addContentFinalizer(content) + } + + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content is bound to snapshot %s", content.Name, snapshotName) + // The VolumeSnapshotContent is reserved for a VolumeSnapshot; + // that VolumeSnapshot has not yet been bound to this VolumeSnapshotContent; the VolumeSnapshot sync will handle it. + if content.Spec.VolumeSnapshotRef.UID == "" { + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is pre-bound to VolumeSnapshot %s", content.Name, snapshotName) + return nil + } + // Get the VolumeSnapshot by _name_ + var snapshot *crdv1.VolumeSnapshot + obj, found, err := ctrl.snapshotStore.GetByKey(snapshotName) + if err != nil { + return err + } + if !found { + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s not found", content.Name, snapshotName) + // Fall through with snapshot = nil + } else { + var ok bool + snapshot, ok = obj.(*crdv1.VolumeSnapshot) + if !ok { + return fmt.Errorf("cannot convert object from snapshot cache to snapshot %q!?: %#v", content.Name, obj) + } + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s found", content.Name, snapshotName) + } + if snapshot != nil && snapshot.UID != content.Spec.VolumeSnapshotRef.UID { + // The snapshot that the content was pointing to was deleted, and another + // with the same name created. + klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content %s has different UID, the old one must have been deleted", content.Name, snapshotName) + // Treat the content as bound to a missing snapshot. + snapshot = nil + } else { + // Check if content status is set to true and update snapshot status if so + if snapshot != nil && content.Status != nil && content.Status.ReadyToUse != nil && *content.Status.ReadyToUse == true { + klog.V(4).Infof("synchronizing VolumeSnapshotContent for snapshot [%s]: update snapshot status to true if needed.", snapshotName) + // Manually trigger a snapshot status update to happen + // right away so that it is in-sync with the content status + ctrl.snapshotQueue.Add(snapshotName) + } + } + + // Trigger content deletion if snapshot has deletion + // timestamp or snapshot does not exist any more + // If snapshot has deletion timestamp and finalizers, set + // AnnVolumeSnapshotBeingDeleted annotation on the content. + // This may trigger the deletion of the content in the + // sidecar controller depending on the deletion policy + // on the content. + // Snapshot won't be deleted until content is deleted + // due to the finalizer + if snapshot == nil || utils.IsSnapshotDeletionCandidate(snapshot) { + // Set AnnVolumeSnapshotBeingDeleted if it is not set yet + if !metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { + klog.V(5).Infof("syncContent: set annotation [%s] on content [%s].", utils.AnnVolumeSnapshotBeingDeleted, content.Name) + metav1.SetMetaDataAnnotation(&content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted, "yes") + + updateContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(content) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(updateContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) + return err + } + klog.V(5).Infof("syncContent: volume snapshot content %+v", content) + } + } + + return nil +} + +// syncSnapshot is the main controller method to decide what to do with a snapshot. +// It's invoked by appropriate cache.Controller callbacks when a snapshot is +// created, updated or periodically synced. We do not differentiate between +// these events. +// For easier readability, it is split into syncUnreadySnapshot and syncReadySnapshot +func (ctrl *csiSnapshotCommonController) syncSnapshot(snapshot *crdv1.VolumeSnapshot) error { + klog.V(5).Infof("synchronizing VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + err := ctrl.processFinalizersAndCheckandDeleteContent(snapshot) + if err != nil { + return err + } + + if !utils.IsSnapshotReady(snapshot) { + return ctrl.syncUnreadySnapshot(snapshot) + } + return ctrl.syncReadySnapshot(snapshot) +} + +// processFinalizersAndCheckandDeleteContent processes finalizers and deletes the content when appropriate +// It checks if contents exists, it checks if snapshot has bi-directional binding, it checks if +// finalizers should be added or removed, and it checks if content should be deleted and deletes it +// if needed. +func (ctrl *csiSnapshotCommonController) processFinalizersAndCheckandDeleteContent(snapshot *crdv1.VolumeSnapshot) error { + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + // If content is deleted already, remove SnapshotBound finalizer + content, err := ctrl.contentExists(snapshot) + if err != nil { + return err + } + deleteContent := false + // It is possible for contentExists to return nil, nil + if content != nil && content.Spec.DeletionPolicy == crdv1.VolumeSnapshotContentDelete { + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent: Content [%s] deletion policy [%s] is delete.", content.Name, content.Spec.DeletionPolicy) + deleteContent = true + } + + snapshotBound := false + // Check if the snapshot content is bound to the snapshot + if content != nil && utils.IsSnapshotBound(snapshot, content) { + klog.Infof("syncSnapshot: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshot.Name, content.Name) + snapshotBound = true + } + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: delete snapshot content and remove finalizer from snapshot if needed", utils.SnapshotKey(snapshot)) + err = ctrl.checkandRemoveSnapshotFinalizersAndCheckandDeleteContent(snapshot, content, deleteContent) + if err != nil { + return err + } + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: check if we should add finalizers on snapshot", utils.SnapshotKey(snapshot)) + ctrl.checkandAddSnapshotFinalizers(snapshot, snapshotBound, deleteContent) + + klog.V(5).Infof("processFinalizersAndCheckandDeleteContent[%s]: check if we should remove finalizer on snapshot source and remove it if we can", utils.SnapshotKey(snapshot)) + + // Check if we should remove finalizer on PVC and remove it if we can. + errFinalizer := ctrl.checkandRemovePVCFinalizer(snapshot) + if errFinalizer != nil { + klog.Errorf("error check and remove PVC finalizer for snapshot [%s]: %v", snapshot.Name, errFinalizer) + // Log an event and keep the original error from syncUnready/ReadySnapshot + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "ErrorPVCFinalizer", "Error check and remove PVC Finalizer for VolumeSnapshot") + } + return nil +} + +// checkandRemoveSnapshotFinalizersAndCheckandDeleteContent deletes the content and removes snapshot finalizers if needed +func (ctrl *csiSnapshotCommonController) checkandRemoveSnapshotFinalizersAndCheckandDeleteContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent, deleteContent bool) error { + klog.V(5).Infof("deleteContentAndSnapshotFinalizers VolumeSnapshot[%s]: %s", utils.SnapshotKey(snapshot), utils.GetSnapshotStatusForLogging(snapshot)) + + var err error + // Check is snapshot deletionTimestamp is set and any finalizer is on + if utils.IsSnapshotDeletionCandidate(snapshot) { + // Volume snapshot should be deleted. Check if it's used + // and remove finalizer if it's not. + // Check if a volume is being created from snapshot. + inUse := ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) + + klog.V(5).Infof("syncSnapshot[%s]: set DeletionTimeStamp on content.", utils.SnapshotKey(snapshot)) + // If content exists, set DeletionTimeStamp on the content; + // content won't be deleted immediately due to the finalizer + if content != nil && deleteContent && !inUse { + err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Delete(content.Name, &metav1.DeleteOptions{}) + if err != nil { + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "SnapshotContentObjectDeleteError", "Failed to delete snapshot content API object") + return fmt.Errorf("failed to delete VolumeSnapshotContent %s from API server: %q", content.Name, err) + + } + } + + if !inUse || (content == nil && err == nil) { + klog.V(5).Infof("syncSnapshot: Remove Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) + doesContentExist := false + if content != nil { + doesContentExist = true + } + return ctrl.removeSnapshotFinalizer(snapshot, !inUse, !doesContentExist) + } + } + return nil +} + +// checkandAddSnapshotFinalizers checks and adds snapshot finailzers when needed +func (ctrl *csiSnapshotCommonController) checkandAddSnapshotFinalizers(snapshot *crdv1.VolumeSnapshot, snapshotBound bool, deleteContent bool) { + addSourceFinalizer := false + addBoundFinalizer := false + if utils.NeedToAddSnapshotAsSourceFinalizer(snapshot) { + addSourceFinalizer = true + } + if utils.NeedToAddSnapshotBoundFinalizer(snapshot) && snapshotBound && deleteContent { + // Add bound finalizer if snapshot is bound to content and deletion policy is delete + addBoundFinalizer = true + } + if addSourceFinalizer || addBoundFinalizer { + // Snapshot is not being deleted -> it should have the finalizer. + klog.V(5).Infof("checkandAddSnapshotFinalizers: Add Finalizer for VolumeSnapshot[%s]", utils.SnapshotKey(snapshot)) + ctrl.addSnapshotFinalizer(snapshot, addSourceFinalizer, addBoundFinalizer) + } +} + +// syncReadySnapshot checks the snapshot which has been bound to snapshot content successfully before. +// If there is any problem with the binding (e.g., snapshot points to a non-exist snapshot content), update the snapshot status and emit event. +func (ctrl *csiSnapshotCommonController) syncReadySnapshot(snapshot *crdv1.VolumeSnapshot) error { + if !utils.IsBoundVolumeSnapshotContentNameSet(snapshot) { + return nil + } + obj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) + if err != nil { + return err + } + if !found { + if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing"); err != nil { + return err + } + return nil + } else { + content, ok := obj.(*crdv1.VolumeSnapshotContent) + if !ok { + return fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", *snapshot.Status.BoundVolumeSnapshotContentName, obj) + } + + klog.V(5).Infof("syncReadySnapshot[%s]: VolumeSnapshotContent %q found", utils.SnapshotKey(snapshot), content.Name) + if !utils.IsVolumeSnapshotRefSet(snapshot, content) { + // snapshot is bound but content is not bound to snapshot correctly + if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotMisbound", "VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"); err != nil { + return err + } + return nil + } + // Snapshot is correctly bound. + return nil + } +} + +// syncUnreadySnapshot is the main controller method to decide what to do with a snapshot which is not set to ready. +func (ctrl *csiSnapshotCommonController) syncUnreadySnapshot(snapshot *crdv1.VolumeSnapshot) error { + uniqueSnapshotName := utils.SnapshotKey(snapshot) + klog.V(5).Infof("syncUnreadySnapshot %s", uniqueSnapshotName) + + // Pre-provisioned snapshot + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + content, err := ctrl.findContentfromStore(snapshot) + if err != nil { + return err + } + // Set VolumeSnapshotRef UID + newContent, err := ctrl.checkandBindSnapshotContent(snapshot, content) + if err != nil { + // snapshot is bound but content is not bound to snapshot correctly + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotBindFailed", fmt.Sprintf("Snapshot failed to bind VolumeSnapshotContent, %v", err)) + return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly, %v", uniqueSnapshotName, content.Name, err) + } + + // update snapshot status + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot)) + _, err = ctrl.updateSnapshotStatus(snapshot, newContent) + if err == nil { + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return err + } + + return nil + } else { // snapshot.Spec.Source.VolumeSnapshotContentName == nil - dynamically creating snapshot + klog.V(5).Infof("before getMatchSnapshotContent for snapshot %s", uniqueSnapshotName) + if contentObj := ctrl.getMatchSnapshotContent(snapshot); contentObj != nil { + klog.V(5).Infof("Found VolumeSnapshotContent object %s for snapshot %s", contentObj.Name, uniqueSnapshotName) + if contentObj.Spec.Source.SnapshotHandle != nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotHandleNotFound", fmt.Sprintf("Snapshot handle not found in content %s", contentObj.Name)) + return fmt.Errorf("snapshotHandle should not be set in the content for dynamic provisioning for snapshot %s", uniqueSnapshotName) + } + newSnapshot, err := ctrl.bindandUpdateVolumeSnapshot(contentObj, snapshot) + if err != nil { + return err + } + klog.V(5).Infof("bindandUpdateVolumeSnapshot %v", newSnapshot) + return nil + } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) + if err != nil { + return err + } + if !found { + if snapshot.ObjectMeta.DeletionTimestamp == nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentNotFound", fmt.Sprintf("Content for snapshot %s not found, but deletion timestamp not set on snapshot", uniqueSnapshotName)) + return fmt.Errorf("content for snapshot %s not found without deletion timestamp on snapshot", uniqueSnapshotName) + } + // NOTE: this is not an error now because we delete content before the snapshot + klog.V(5).Infof("Content for snapshot %s not found. It may be already deleted as expected.", uniqueSnapshotName) + } else { + klog.V(5).Infof("converting content object for snapshot %s", uniqueSnapshotName) + _, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + return fmt.Errorf("expected volume snapshot content, got %+v", contentObj) + } + } + } else if snapshot.Status == nil || snapshot.Status.Error == nil || isControllerUpdateFailError(snapshot.Status.Error) { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotPVCSourceMissing", fmt.Sprintf("PVC source for snapshot %s is missing", uniqueSnapshotName)) + return fmt.Errorf("expected PVC source for snapshot %s but got nil", uniqueSnapshotName) + } else { + var err error + var content *crdv1.VolumeSnapshotContent + if content, err = ctrl.createSnapshotContent(snapshot); err != nil { + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentCreationFailed", fmt.Sprintf("Failed to create snapshot content with error %v", err)) + return err + } + + // Update snapshot status with BoundVolumeSnapshotContentName + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("syncUnreadySnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshot)) + _, err = ctrl.updateSnapshotStatus(snapshot, content) + if err == nil { + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return err + } + } + } + return nil + } +} + +// findContentfromStore finds content from content cache store +func (ctrl *csiSnapshotCommonController) findContentfromStore(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + var contentName string + uniqueSnapshotName := utils.SnapshotKey(snapshot) + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + contentName = *snapshot.Spec.Source.VolumeSnapshotContentName + } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + if contentName == "" { + return nil, fmt.Errorf("content name not found for snapshot %s", uniqueSnapshotName) + } + + contentObj, found, err := ctrl.contentStore.GetByKey(contentName) + if err != nil { + return nil, err + } + if !found { + // snapshot is bound to a non-existing content. + ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") + klog.V(4).Infof("synchronizing unready snapshot[%s]: snapshotcontent %q requested and not found, will try again next time", uniqueSnapshotName, contentName) + return nil, fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, contentName) + } + content, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + return nil, fmt.Errorf("expected volume snapshot content, got %+v", contentObj) + } + return content, nil +} + +// createSnapshotContent will only be called for dynamic provisioning +func (ctrl *csiSnapshotCommonController) createSnapshotContent(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + klog.Infof("createSnapshotContent: Creating content for snapshot %s through the plugin ...", utils.SnapshotKey(snapshot)) + + // If PVC is not being deleted and finalizer is not added yet, a finalizer should be added to PVC until snapshot is created + klog.V(5).Infof("createSnapshotContent: Check if PVC is not being deleted and add Finalizer for source of snapshot [%s] if needed", snapshot.Name) + err := ctrl.ensurePVCFinalizer(snapshot) + if err != nil { + klog.Errorf("createSnapshotContent failed to add finalizer for source of snapshot %s", err) + return nil, err + } + + class, volume, contentName, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) + } + + // Create VolumeSnapshotContent in the database + if volume.Spec.CSI == nil { + return nil, fmt.Errorf("cannot find CSI PersistentVolumeSource for volume %s", volume.Name) + } + snapshotRef, err := ref.GetReference(scheme.Scheme, snapshot) + if err != nil { + return nil, err + } + + snapshotContent := &crdv1.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: contentName, + }, + Spec: crdv1.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: *snapshotRef, + Source: crdv1.VolumeSnapshotContentSource{ + VolumeHandle: &volume.Spec.CSI.VolumeHandle, + }, + VolumeSnapshotClassName: &(class.Name), + DeletionPolicy: class.DeletionPolicy, + Driver: class.Driver, + }, + } + + // Set AnnDeletionSecretRefName and AnnDeletionSecretRefNamespace + if snapshotterSecretRef != nil { + klog.V(5).Infof("createSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefName, snapshotContent.Name) + metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, utils.AnnDeletionSecretRefName, snapshotterSecretRef.Name) + + klog.V(5).Infof("createSnapshotContent: set annotation [%s] on content [%s].", utils.AnnDeletionSecretRefNamespace, snapshotContent.Name) + metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, utils.AnnDeletionSecretRefNamespace, snapshotterSecretRef.Namespace) + } + + var updateContent *crdv1.VolumeSnapshotContent + klog.V(3).Infof("volume snapshot content %#v", snapshotContent) + // Try to create the VolumeSnapshotContent object several times + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("createSnapshotContent [%s]: trying to save volume snapshot content %s", utils.SnapshotKey(snapshot), snapshotContent.Name) + if updateContent, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) { + // Save succeeded. + if err != nil { + klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, utils.SnapshotKey(snapshot)) + err = nil + updateContent = snapshotContent + } else { + klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, utils.SnapshotKey(snapshot), snapshotContent) + } + break + } + // Save failed, try again after a while. + klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, utils.SnapshotKey(snapshot), err) + time.Sleep(ctrl.createSnapshotContentInterval) + } + + if err != nil { + strerr := fmt.Sprintf("Error creating volume snapshot content object for snapshot %s: %v.", utils.SnapshotKey(snapshot), err) + klog.Error(strerr) + ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "CreateSnapshotContentFailed", strerr) + return nil, newControllerUpdateError(utils.SnapshotKey(snapshot), err.Error()) + } + + // Update content in the cache store + _, err = ctrl.storeContentUpdate(updateContent) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + return updateContent, nil +} + +func (ctrl *csiSnapshotCommonController) getCreateSnapshotInput(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *v1.PersistentVolume, string, *v1.SecretReference, error) { + className := snapshot.Spec.VolumeSnapshotClassName + klog.V(5).Infof("getCreateSnapshotInput [%s]", snapshot.Name) + var class *crdv1.VolumeSnapshotClass + var err error + if className != nil { + class, err = ctrl.getSnapshotClass(*className) + if err != nil { + klog.Errorf("getCreateSnapshotInput failed to getClassFromVolumeSnapshot %s", err) + return nil, nil, "", nil, err + } + } else { + klog.Errorf("failed to getCreateSnapshotInput %s without a snapshot class", snapshot.Name) + return nil, nil, "", nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", snapshot.Name) + } + + volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) + if err != nil { + klog.Errorf("getCreateSnapshotInput failed to get PersistentVolume object [%s]: Error: [%#v]", snapshot.Name, err) + return nil, nil, "", nil, err + } + + // Create VolumeSnapshotContent name + contentName := utils.GetSnapshotContentNameForSnapshot(snapshot) + + // Resolve snapshotting secret credentials. + snapshotterSecretRef, err := utils.GetSecretReference(class.Parameters, contentName, snapshot) + if err != nil { + return nil, nil, "", nil, err + } + + return class, volume, contentName, snapshotterSecretRef, nil +} + +// getMatchSnapshotContent looks up VolumeSnapshotContent for a VolumeSnapshot named snapshotName +func (ctrl *csiSnapshotCommonController) getMatchSnapshotContent(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshotContent { + var snapshotContentObj *crdv1.VolumeSnapshotContent + var found bool + + objs := ctrl.contentStore.List() + for _, obj := range objs { + content := obj.(*crdv1.VolumeSnapshotContent) + if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && + content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && + content.Spec.VolumeSnapshotRef.UID == snapshot.UID && + content.Spec.VolumeSnapshotClassName != nil && snapshot.Spec.VolumeSnapshotClassName != nil && + *(content.Spec.VolumeSnapshotClassName) == *(snapshot.Spec.VolumeSnapshotClassName) { + found = true + snapshotContentObj = content + break + } + } + + if !found { + klog.V(4).Infof("No VolumeSnapshotContent for VolumeSnapshot %s found", utils.SnapshotKey(snapshot)) + return nil + } + + return snapshotContentObj +} + +func (ctrl *csiSnapshotCommonController) storeSnapshotUpdate(snapshot interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.snapshotStore, snapshot, "snapshot") +} + +func (ctrl *csiSnapshotCommonController) storeContentUpdate(content interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.contentStore, content, "content") +} + +// updateSnapshotStatusWithEvent saves new snapshot.Status to API server and emits +// given event on the snapshot. It saves the status and emits the event only when +// the status has actually changed from the version saved in API server. +// Parameters: +// snapshot - snapshot to update +// eventtype, reason, message - event to send, see EventRecorder.Event() +func (ctrl *csiSnapshotCommonController) updateSnapshotErrorStatusWithEvent(snapshot *crdv1.VolumeSnapshot, eventtype, reason, message string) error { + klog.V(5).Infof("updateSnapshotStatusWithEvent[%s]", utils.SnapshotKey(snapshot)) + + if snapshot.Status != nil && snapshot.Status.Error != nil && *snapshot.Status.Error.Message == message { + klog.V(4).Infof("updateSnapshotStatusWithEvent[%s]: the same error %v is already set", snapshot.Name, snapshot.Status.Error) + return nil + } + snapshotClone := snapshot.DeepCopy() + if snapshotClone.Status == nil { + snapshotClone.Status = &crdv1.VolumeSnapshotStatus{} + } + statusError := &crdv1.VolumeSnapshotError{ + Time: &metav1.Time{ + Time: time.Now(), + }, + Message: &message, + } + snapshotClone.Status.Error = statusError + ready := false + snapshotClone.Status.ReadyToUse = &ready + newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) + + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] error status failed %v", utils.SnapshotKey(snapshot), err) + return err + } + + _, err = ctrl.storeSnapshotUpdate(newSnapshot) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", utils.SnapshotKey(snapshot), err) + return err + } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) + + return nil +} + +// isSnapshotConentBeingUsed checks if snapshot content is bound to snapshot. +func (ctrl *csiSnapshotCommonController) isSnapshotContentBeingUsed(content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.VolumeSnapshotRef.Name != "" && content.Spec.VolumeSnapshotRef.Namespace != "" { + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(content.Spec.VolumeSnapshotRef.Namespace).Get(content.Spec.VolumeSnapshotRef.Name, metav1.GetOptions{}) + if err != nil { + klog.Infof("isSnapshotContentBeingUsed: Cannot get snapshot %s from api server: [%v]. VolumeSnapshot object may be deleted already.", content.Spec.VolumeSnapshotRef.Name, err) + return false + } + + // Check if the snapshot content is bound to the snapshot + if utils.IsSnapshotBound(snapshotObj, content) { + klog.Infof("isSnapshotContentBeingUsed: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshotObj.Name, content.Name) + return true + } + } + + klog.V(5).Infof("isSnapshotContentBeingUsed: Snapshot content %s is not being used", content.Name) + return false +} + +// addContentFinalizer adds a Finalizer for VolumeSnapshotContent. +func (ctrl *csiSnapshotCommonController) addContentFinalizer(content *crdv1.VolumeSnapshotContent) error { + contentClone := content.DeepCopy() + contentClone.ObjectMeta.Finalizers = append(contentClone.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer) + + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(contentClone) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume snapshot content %s", content.Name) + return nil +} + +// isVolumeBeingCreatedFromSnapshot checks if an volume is being created from the snapshot. +func (ctrl *csiSnapshotCommonController) isVolumeBeingCreatedFromSnapshot(snapshot *crdv1.VolumeSnapshot) bool { + pvcList, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).List(labels.Everything()) + if err != nil { + klog.Errorf("Failed to retrieve PVCs from the lister to check if volume snapshot %s is being used by a volume: %q", utils.SnapshotKey(snapshot), err) + return false + } + for _, pvc := range pvcList { + if pvc.Spec.DataSource != nil && pvc.Spec.DataSource.Name == snapshot.Name { + if pvc.Spec.DataSource.Kind == snapshotKind && *(pvc.Spec.DataSource.APIGroup) == snapshotAPIGroup { + if pvc.Status.Phase == v1.ClaimPending { + // A volume is being created from the snapshot + klog.Infof("isVolumeBeingCreatedFromSnapshot: volume %s is being created from snapshot %s", pvc.Name, pvc.Spec.DataSource.Name) + return true + } + } + } + } + klog.V(5).Infof("isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot %s", utils.SnapshotKey(snapshot)) + return false +} + +// ensurePVCFinalizer checks if a Finalizer needs to be added for the snapshot source; +// if true, adds a Finalizer for VolumeSnapshot Source PVC +func (ctrl *csiSnapshotCommonController) ensurePVCFinalizer(snapshot *crdv1.VolumeSnapshot) error { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + // PVC finalizer is only needed for dynamic provisioning + return nil + } + + // Get snapshot source which is a PVC + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) + return nil + } + + if pvc.ObjectMeta.DeletionTimestamp != nil { + klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: claim is being deleted", pvc.Name, snapshot.Name) + return newControllerUpdateError(pvc.Name, "cannot add finalizer on claim because it is being deleted") + } + + // If PVC is not being deleted and PVCFinalizer is not added yet, the PVCFinalizer should be added. + if pvc.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { + // Add the finalizer + pvcClone := pvc.DeepCopy() + pvcClone.ObjectMeta.Finalizers = append(pvcClone.ObjectMeta.Finalizers, utils.PVCFinalizer) + _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) + if err != nil { + klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: [%v]", pvc.Name, snapshot.Name, err) + return newControllerUpdateError(pvcClone.Name, err.Error()) + } + klog.Infof("Added protection finalizer to persistent volume claim %s", pvc.Name) + } + + return nil +} + +// removePVCFinalizer removes a Finalizer for VolumeSnapshot Source PVC. +func (ctrl *csiSnapshotCommonController) removePVCFinalizer(pvc *v1.PersistentVolumeClaim, snapshot *crdv1.VolumeSnapshot) error { + // Get snapshot source which is a PVC + // TODO(xyang): We get PVC from informer but it may be outdated + // Should get it from API server directly before removing finalizer + pvcClone := pvc.DeepCopy() + pvcClone.ObjectMeta.Finalizers = slice.RemoveString(pvcClone.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) + + _, err := ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) + if err != nil { + return newControllerUpdateError(pvcClone.Name, err.Error()) + } + + klog.V(5).Infof("Removed protection finalizer from persistent volume claim %s", pvc.Name) + return nil +} + +// isPVCBeingUsed checks if a PVC is being used as a source to create a snapshot +func (ctrl *csiSnapshotCommonController) isPVCBeingUsed(pvc *v1.PersistentVolumeClaim, snapshot *crdv1.VolumeSnapshot) bool { + klog.V(5).Infof("Checking isPVCBeingUsed for snapshot [%s]", utils.SnapshotKey(snapshot)) + + // Going through snapshots in the cache (snapshotLister). If a snapshot's PVC source + // is the same as the input snapshot's PVC source and snapshot's ReadyToUse status + // is false, the snapshot is still being created from the PVC and the PVC is in-use. + snapshots, err := ctrl.snapshotLister.VolumeSnapshots(snapshot.Namespace).List(labels.Everything()) + if err != nil { + return false + } + for _, snap := range snapshots { + // Skip pre-provisioned snapshot without a PVC source + if snap.Spec.Source.PersistentVolumeClaimName == nil && snap.Spec.Source.VolumeSnapshotContentName != nil { + klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) + continue + } + if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && (snap.Status == nil || snap.Status.ReadyToUse == nil || (snap.Status.ReadyToUse != nil && *snap.Status.ReadyToUse == false)) { + klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) + return true + } + } + + klog.V(5).Infof("isPVCBeingUsed: no snapshot is being created from PVC %s/%s", pvc.Namespace, pvc.Name) + return false +} + +// checkandRemovePVCFinalizer checks if the snapshot source finalizer should be removed +// and removed it if needed. +func (ctrl *csiSnapshotCommonController) checkandRemovePVCFinalizer(snapshot *crdv1.VolumeSnapshot) error { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + // PVC finalizer is only needed for dynamic provisioning + return nil + } + + // Get snapshot source which is a PVC + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) + return nil + } + + klog.V(5).Infof("checkandRemovePVCFinalizer for snapshot [%s]: snapshot status [%#v]", snapshot.Name, snapshot.Status) + + // Check if there is a Finalizer on PVC to be removed + if slice.ContainsString(pvc.ObjectMeta.Finalizers, utils.PVCFinalizer, nil) { + // There is a Finalizer on PVC. Check if PVC is used + // and remove finalizer if it's not used. + inUse := ctrl.isPVCBeingUsed(pvc, snapshot) + if !inUse { + klog.Infof("checkandRemovePVCFinalizer[%s]: Remove Finalizer for PVC %s as it is not used by snapshots in creation", snapshot.Name, pvc.Name) + err = ctrl.removePVCFinalizer(pvc, snapshot) + if err != nil { + klog.Errorf("checkandRemovePVCFinalizer [%s]: removePVCFinalizer failed to remove finalizer %v", snapshot.Name, err) + return err + } + } + } + + return nil +} + +// The function checks whether the volumeSnapshotRef in snapshot content matches the given snapshot. If match, it binds the content with the snapshot. This is for static binding where user has specified snapshot name but not UID of the snapshot in content.Spec.VolumeSnapshotRef. +func (ctrl *csiSnapshotCommonController) checkandBindSnapshotContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + if content.Spec.VolumeSnapshotRef.Name != snapshot.Name { + return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) + } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotRef.UID != snapshot.UID { + return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) + } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotClassName != nil { + return content, nil + } + contentClone := content.DeepCopy() + contentClone.Spec.VolumeSnapshotRef.UID = snapshot.UID + if snapshot.Spec.VolumeSnapshotClassName != nil { + className := *(snapshot.Spec.VolumeSnapshotClassName) + contentClone.Spec.VolumeSnapshotClassName = &className + } + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", newContent.Name, err) + return nil, err + } + + _, err = ctrl.storeContentUpdate(newContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", newContent.Name, err) + return nil, err + } + return newContent, nil +} + +// This routine sets snapshot.Spec.Source.VolumeSnapshotContentName +func (ctrl *csiSnapshotCommonController) bindandUpdateVolumeSnapshot(snapshotContent *crdv1.VolumeSnapshotContent, snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("bindandUpdateVolumeSnapshot for snapshot [%s]: snapshotContent [%s]", snapshot.Name, snapshotContent.Name) + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshot.Namespace).Get(snapshot.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot %s from api server: %v", utils.SnapshotKey(snapshot), err) + } + + // Copy the snapshot object before updating it + snapshotCopy := snapshotObj.DeepCopy() + + // update snapshot status + for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { + klog.V(5).Infof("bindandUpdateVolumeSnapshot [%s]: trying to update snapshot status", utils.SnapshotKey(snapshotCopy)) + updateSnapshot, err := ctrl.updateSnapshotStatus(snapshotCopy, snapshotContent) + if err == nil { + snapshotCopy = updateSnapshot + break + } + klog.V(4).Infof("failed to update snapshot %s status: %v", utils.SnapshotKey(snapshot), err) + } + + if err != nil { + // update snapshot status failed + ctrl.updateSnapshotErrorStatusWithEvent(snapshotCopy, v1.EventTypeWarning, "SnapshotStatusUpdateFailed", fmt.Sprintf("Snapshot status update failed, %v", err)) + return nil, err + } + + _, err = ctrl.storeSnapshotUpdate(snapshotCopy) + if err != nil { + klog.Errorf("%v", err) + } + + klog.V(5).Infof("bindandUpdateVolumeSnapshot for snapshot completed [%#v]", snapshotCopy) + return snapshotCopy, nil +} + +// UpdateSnapshotStatus updates snapshot status based on content status +func (ctrl *csiSnapshotCommonController) updateSnapshotStatus(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("updateSnapshotStatus[%s]", utils.SnapshotKey(snapshot)) + + boundContentName := content.Name + var createdAt *time.Time + if content.Status != nil && content.Status.CreationTime != nil { + unixTime := time.Unix(0, *content.Status.CreationTime) + createdAt = &unixTime + } + var size *int64 + if content.Status != nil && content.Status.RestoreSize != nil { + size = content.Status.RestoreSize + } + var readyToUse bool + if content.Status != nil && content.Status.ReadyToUse != nil { + readyToUse = *content.Status.ReadyToUse + } + + klog.V(5).Infof("updateSnapshotStatus: updating VolumeSnapshot [%+v] based on VolumeSnapshotContentStatus [%+v]", snapshot, content.Status) + + snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshot.Namespace).Get(snapshot.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot %s from api server: %v", utils.SnapshotKey(snapshot), err) + } + + var newStatus *crdv1.VolumeSnapshotStatus + updated := false + if snapshotObj.Status == nil { + newStatus = &crdv1.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &boundContentName, + ReadyToUse: &readyToUse, + } + if createdAt != nil { + newStatus.CreationTime = &metav1.Time{Time: *createdAt} + } + if size != nil { + newStatus.RestoreSize = resource.NewQuantity(*size, resource.BinarySI) + } + updated = true + } else { + newStatus = snapshotObj.Status.DeepCopy() + if newStatus.BoundVolumeSnapshotContentName == nil { + newStatus.BoundVolumeSnapshotContentName = &boundContentName + updated = true + } + if newStatus.CreationTime == nil && createdAt != nil { + newStatus.CreationTime = &metav1.Time{Time: *createdAt} + updated = true + } + if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { + newStatus.ReadyToUse = &readyToUse + updated = true + if readyToUse && newStatus.Error != nil { + newStatus.Error = nil + } + } + if (newStatus.RestoreSize == nil && size != nil) || (newStatus.RestoreSize != nil && newStatus.RestoreSize.IsZero() && size != nil && *size > 0) { + newStatus.RestoreSize = resource.NewQuantity(*size, resource.BinarySI) + updated = true + } + } + + if updated { + snapshotClone := snapshotObj.DeepCopy() + snapshotClone.Status = newStatus + newSnapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) + if err != nil { + return nil, newControllerUpdateError(utils.SnapshotKey(snapshot), err.Error()) + } + return newSnapshotObj, nil + } + + return snapshotObj, nil +} + +func (ctrl *csiSnapshotCommonController) getVolumeFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolume, error) { + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + + if pvc.Status.Phase != v1.ClaimBound { + return nil, fmt.Errorf("the PVC %s is not yet bound to a PV, will not attempt to take a snapshot", pvc.Name) + } + + pvName := pvc.Spec.VolumeName + pv, err := ctrl.client.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to retrieve PV %s from the API server: %q", pvName, err) + } + + klog.V(5).Infof("getVolumeFromVolumeSnapshot: snapshot [%s] PV name [%s]", snapshot.Name, pvName) + + return pv, nil +} + +func (ctrl *csiSnapshotCommonController) getStorageClassFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*storagev1.StorageClass, error) { + // Get storage class from PVC or PV + pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + storageclassName := *pvc.Spec.StorageClassName + if len(storageclassName) == 0 { + volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) + if err != nil { + return nil, err + } + storageclassName = volume.Spec.StorageClassName + } + if len(storageclassName) == 0 { + return nil, fmt.Errorf("cannot figure out the snapshot class automatically, please specify one in snapshot spec") + } + storageclass, err := ctrl.client.StorageV1().StorageClasses().Get(storageclassName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return storageclass, nil +} + +// getSnapshotClass is a helper function to get snapshot class from the class name. +func (ctrl *csiSnapshotCommonController) getSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { + klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) + + class, err := ctrl.classLister.Get(className) + if err != nil { + klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + } + + return class, nil +} + +// SetDefaultSnapshotClass is a helper function to figure out the default snapshot class from +// PVC/PV StorageClass and update VolumeSnapshot with this snapshot class name. +func (ctrl *csiSnapshotCommonController) SetDefaultSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *crdv1.VolumeSnapshot, error) { + klog.V(5).Infof("SetDefaultSnapshotClass for snapshot [%s]", snapshot.Name) + + if snapshot.Spec.Source.VolumeSnapshotContentName != nil { + // don't return error for pre-provisioned snapshots + klog.V(5).Infof("Don't need to find SnapshotClass for pre-provisioned snapshot [%s]", snapshot.Name) + return nil, snapshot, nil + } + + storageclass, err := ctrl.getStorageClassFromVolumeSnapshot(snapshot) + if err != nil { + return nil, nil, err + } + // Find default snapshot class if available + list, err := ctrl.classLister.List(labels.Everything()) + if err != nil { + return nil, nil, err + } + defaultClasses := []*crdv1.VolumeSnapshotClass{} + + for _, class := range list { + if utils.IsDefaultAnnotation(class.ObjectMeta) && storageclass.Provisioner == class.Driver { //&& ctrl.snapshotterName == class.Snapshotter { + defaultClasses = append(defaultClasses, class) + klog.V(5).Infof("get defaultClass added: %s", class.Name) + } + } + if len(defaultClasses) == 0 { + return nil, nil, fmt.Errorf("cannot find default snapshot class") + } + if len(defaultClasses) > 1 { + klog.V(4).Infof("get DefaultClass %d defaults found", len(defaultClasses)) + return nil, nil, fmt.Errorf("%d default snapshot classes were found", len(defaultClasses)) + } + klog.V(5).Infof("setDefaultSnapshotClass [%s]: default VolumeSnapshotClassName [%s]", snapshot.Name, defaultClasses[0].Name) + snapshotClone := snapshot.DeepCopy() + snapshotClone.Spec.VolumeSnapshotClassName = &(defaultClasses[0].Name) + newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshot[%s] default class failed %v", utils.SnapshotKey(snapshot), err) + } + _, updateErr := ctrl.storeSnapshotUpdate(newSnapshot) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("setDefaultSnapshotClass [%s]: cannot update internal cache: %v", utils.SnapshotKey(snapshot), updateErr) + } + + return defaultClasses[0], newSnapshot, nil +} + +// getClaimFromVolumeSnapshot is a helper function to get PVC from VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) getClaimFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolumeClaim, error) { + if snapshot.Spec.Source.PersistentVolumeClaimName == nil { + return nil, fmt.Errorf("the snapshot source PVC name is not specified") + } + pvcName := *snapshot.Spec.Source.PersistentVolumeClaimName + if pvcName == "" { + return nil, fmt.Errorf("the PVC name is not specified in snapshot %s", utils.SnapshotKey(snapshot)) + } + + pvc, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).Get(pvcName) + if err != nil { + return nil, fmt.Errorf("failed to retrieve PVC %s from the lister: %q", pvcName, err) + } + + return pvc, nil +} + +var _ error = controllerUpdateError{} + +type controllerUpdateError struct { + message string +} + +func newControllerUpdateError(name, message string) error { + return controllerUpdateError{ + message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), + } +} + +func (e controllerUpdateError) Error() string { + return e.message +} + +func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { + if err != nil { + if strings.Contains(*err.Message, controllerUpdateFailMsg) { + return true + } + } + return false +} + +// addSnapshotFinalizer adds a Finalizer for VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) addSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, addSourceFinalizer bool, addBoundFinalizer bool) error { + snapshotClone := snapshot.DeepCopy() + if addSourceFinalizer { + snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer) + } + if addBoundFinalizer { + snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer) + } + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + return newControllerUpdateError(snapshot.Name, err.Error()) + } + + _, err = ctrl.storeSnapshotUpdate(snapshotClone) + if err != nil { + klog.Errorf("failed to update snapshot store %v", err) + } + + klog.V(5).Infof("Added protection finalizer to volume snapshot %s", utils.SnapshotKey(snapshot)) + return nil +} + +// removeSnapshotFinalizer removes a Finalizer for VolumeSnapshot. +func (ctrl *csiSnapshotCommonController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, removeSourceFinalizer bool, removeBoundFinalizer bool) error { + snapshotClone := snapshot.DeepCopy() + if removeSourceFinalizer { + snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer, nil) + } + if removeBoundFinalizer { + snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotBoundFinalizer, nil) + } + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) + if err != nil { + return newControllerUpdateError(snapshot.Name, err.Error()) + } + + _, err = ctrl.storeSnapshotUpdate(snapshotClone) + if err != nil { + klog.Errorf("failed to update snapshot store %v", err) + } + + klog.V(5).Infof("Removed protection finalizer from volume snapshot %s", utils.SnapshotKey(snapshot)) + return nil +} + +func (ctrl *csiSnapshotCommonController) contentExists(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotContent, error) { + var contentName string + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + contentName = *snapshot.Status.BoundVolumeSnapshotContentName + } else { + contentName = utils.GetSnapshotContentNameForSnapshot(snapshot) + } + obj, found, err := ctrl.contentStore.GetByKey(contentName) + if err != nil { + return nil, err + } + // Not in the content cache store, no error + if !found { + return nil, nil + } + // Found in content cache store + content, ok := obj.(*crdv1.VolumeSnapshotContent) + if !ok { + return content, fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", contentName, obj) + } + // Found in content cache store and convert object is successful + return content, nil +} diff --git a/pkg/controller/snapshot_controller_base.go b/pkg/common-controller/snapshot_controller_base.go similarity index 80% rename from pkg/controller/snapshot_controller_base.go rename to pkg/common-controller/snapshot_controller_base.go index 7f58554ca..958f03fae 100644 --- a/pkg/controller/snapshot_controller_base.go +++ b/pkg/common-controller/snapshot_controller_base.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "fmt" @@ -24,9 +24,9 @@ import ( clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" - "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -42,10 +42,9 @@ import ( "k8s.io/kubernetes/pkg/util/goroutinemap" ) -type csiSnapshotController struct { +type csiSnapshotCommonController struct { clientset clientset.Interface client kubernetes.Interface - driverName string eventRecorder record.EventRecorder snapshotQueue workqueue.RateLimitingInterface contentQueue workqueue.RateLimitingInterface @@ -62,7 +61,6 @@ type csiSnapshotController struct { snapshotStore cache.Store contentStore cache.Store - handler Handler // Map of scheduled/running operations. runningOperations goroutinemap.GoRoutineMap @@ -71,43 +69,36 @@ type csiSnapshotController struct { resyncPeriod time.Duration } -// NewCSISnapshotController returns a new *csiSnapshotController -func NewCSISnapshotController( +// NewCSISnapshotController returns a new *csiSnapshotCommonController +func NewCSISnapshotCommonController( clientset clientset.Interface, client kubernetes.Interface, - driverName string, volumeSnapshotInformer storageinformers.VolumeSnapshotInformer, volumeSnapshotContentInformer storageinformers.VolumeSnapshotContentInformer, volumeSnapshotClassInformer storageinformers.VolumeSnapshotClassInformer, pvcInformer coreinformers.PersistentVolumeClaimInformer, createSnapshotContentRetryCount int, createSnapshotContentInterval time.Duration, - snapshotter snapshotter.Snapshotter, - timeout time.Duration, resyncPeriod time.Duration, - snapshotNamePrefix string, - snapshotNameUUIDLength int, -) *csiSnapshotController { +) *csiSnapshotCommonController { broadcaster := record.NewBroadcaster() broadcaster.StartLogging(klog.Infof) broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) var eventRecorder record.EventRecorder - eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("csi-snapshotter %s", driverName)}) + eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("snapshot-controller")}) - ctrl := &csiSnapshotController{ + ctrl := &csiSnapshotCommonController{ clientset: clientset, client: client, - driverName: driverName, eventRecorder: eventRecorder, - handler: NewCSIHandler(snapshotter, timeout, snapshotNamePrefix, snapshotNameUUIDLength), runningOperations: goroutinemap.NewGoRoutineMap(true), createSnapshotContentRetryCount: createSnapshotContentRetryCount, createSnapshotContentInterval: createSnapshotContentInterval, resyncPeriod: resyncPeriod, snapshotStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), - snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-snapshot"), - contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-content"), + snapshotQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-snapshot"), + contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "snapshot-controller-content"), } ctrl.pvcLister = pvcInformer.Lister() @@ -141,12 +132,12 @@ func NewCSISnapshotController( return ctrl } -func (ctrl *csiSnapshotController) Run(workers int, stopCh <-chan struct{}) { +func (ctrl *csiSnapshotCommonController) Run(workers int, stopCh <-chan struct{}) { defer ctrl.snapshotQueue.ShutDown() defer ctrl.contentQueue.ShutDown() - klog.Infof("Starting CSI snapshotter") - defer klog.Infof("Shutting CSI snapshotter") + klog.Infof("Starting snapshot controller") + defer klog.Infof("Shutting snapshot controller") if !cache.WaitForCacheSync(stopCh, ctrl.snapshotListerSynced, ctrl.contentListerSynced, ctrl.classListerSynced, ctrl.pvcListerSynced) { klog.Errorf("Cannot sync caches") @@ -164,7 +155,7 @@ func (ctrl *csiSnapshotController) Run(workers int, stopCh <-chan struct{}) { } // enqueueSnapshotWork adds snapshot to given work queue. -func (ctrl *csiSnapshotController) enqueueSnapshotWork(obj interface{}) { +func (ctrl *csiSnapshotCommonController) enqueueSnapshotWork(obj interface{}) { // Beware of "xxx deleted" events if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { obj = unknown.Obj @@ -181,7 +172,7 @@ func (ctrl *csiSnapshotController) enqueueSnapshotWork(obj interface{}) { } // enqueueContentWork adds snapshot content to given work queue. -func (ctrl *csiSnapshotController) enqueueContentWork(obj interface{}) { +func (ctrl *csiSnapshotCommonController) enqueueContentWork(obj interface{}) { // Beware of "xxx deleted" events if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { obj = unknown.Obj @@ -199,7 +190,7 @@ func (ctrl *csiSnapshotController) enqueueContentWork(obj interface{}) { // snapshotWorker processes items from snapshotQueue. It must run only once, // syncSnapshot is not assured to be reentrant. -func (ctrl *csiSnapshotController) snapshotWorker() { +func (ctrl *csiSnapshotCommonController) snapshotWorker() { workFunc := func() bool { keyObj, quit := ctrl.snapshotQueue.Get() if quit { @@ -264,7 +255,7 @@ func (ctrl *csiSnapshotController) snapshotWorker() { // contentWorker processes items from contentQueue. It must run only once, // syncContent is not assured to be reentrant. -func (ctrl *csiSnapshotController) contentWorker() { +func (ctrl *csiSnapshotCommonController) contentWorker() { workFunc := func() bool { keyObj, quit := ctrl.contentQueue.Get() if quit { @@ -283,9 +274,7 @@ func (ctrl *csiSnapshotController) contentWorker() { // The content still exists in informer cache, the event must have // been add/update/sync if err == nil { - if ctrl.isDriverMatch(content) { - ctrl.updateContent(content) - } + ctrl.updateContent(content) return false } if !errors.IsNotFound(err) { @@ -323,22 +312,16 @@ func (ctrl *csiSnapshotController) contentWorker() { } } -// verify whether the driver specified in VolumeSnapshotContent matches the controller's driver name -func (ctrl *csiSnapshotController) isDriverMatch(content *crdv1.VolumeSnapshotContent) bool { - return content.Spec.Driver == ctrl.driverName -} - // checkAndUpdateSnapshotClass gets the VolumeSnapshotClass from VolumeSnapshot. If it is not set, -// gets it from default VolumeSnapshotClass and sets it. It also detects if snapshotter in the -// VolumeSnapshotClass is the same as the snapshotter in external controller. -func (ctrl *csiSnapshotController) checkAndUpdateSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { +// gets it from default VolumeSnapshotClass and sets it. +func (ctrl *csiSnapshotCommonController) checkAndUpdateSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { className := snapshot.Spec.VolumeSnapshotClassName var class *crdv1.VolumeSnapshotClass var err error newSnapshot := snapshot if className != nil { klog.V(5).Infof("checkAndUpdateSnapshotClass [%s]: VolumeSnapshotClassName [%s]", snapshot.Name, *className) - class, err = ctrl.GetSnapshotClass(*className) + class, err = ctrl.getSnapshotClass(*className) if err != nil { klog.Errorf("checkAndUpdateSnapshotClass failed to getSnapshotClass %v", err) ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "GetSnapshotClassFailed", fmt.Sprintf("Failed to get snapshot class with error %v", err)) @@ -354,20 +337,19 @@ func (ctrl *csiSnapshotController) checkAndUpdateSnapshotClass(snapshot *crdv1.V } } - klog.V(5).Infof("VolumeSnapshotClass Driver [%s] Snapshot Controller driverName [%s]", class.Driver, ctrl.driverName) - if class.Driver != ctrl.driverName { - klog.V(4).Infof("Skipping VolumeSnapshot %s for snapshotter [%s] in VolumeSnapshotClass because it does not match with the snapshotter for controller [%s]", snapshotKey(snapshot), class.Driver, ctrl.driverName) - return nil, fmt.Errorf("volumeSnapshotClass does not match with the snapshotter for controller") + // For pre-provisioned snapshots, we may not have snapshot class + if class != nil { + klog.V(5).Infof("VolumeSnapshotClass [%s] Driver [%s]", class.Name, class.Driver) } return newSnapshot, nil } // updateSnapshot runs in worker thread and handles "snapshot added", // "snapshot updated" and "periodic sync" events. -func (ctrl *csiSnapshotController) updateSnapshot(snapshot *crdv1.VolumeSnapshot) { +func (ctrl *csiSnapshotCommonController) updateSnapshot(snapshot *crdv1.VolumeSnapshot) { // Store the new snapshot version in the cache and do not process it if this is // an old version. - klog.V(5).Infof("updateSnapshot %q", snapshotKey(snapshot)) + klog.V(5).Infof("updateSnapshot %q", utils.SnapshotKey(snapshot)) newSnapshot, err := ctrl.storeSnapshotUpdate(snapshot) if err != nil { klog.Errorf("%v", err) @@ -380,16 +362,16 @@ func (ctrl *csiSnapshotController) updateSnapshot(snapshot *crdv1.VolumeSnapshot if errors.IsConflict(err) { // Version conflict error happens quite often and the controller // recovers from it easily. - klog.V(3).Infof("could not sync claim %q: %+v", snapshotKey(snapshot), err) + klog.V(3).Infof("could not sync claim %q: %+v", utils.SnapshotKey(snapshot), err) } else { - klog.Errorf("could not sync volume %q: %+v", snapshotKey(snapshot), err) + klog.Errorf("could not sync volume %q: %+v", utils.SnapshotKey(snapshot), err) } } } // updateContent runs in worker thread and handles "content added", // "content updated" and "periodic sync" events. -func (ctrl *csiSnapshotController) updateContent(content *crdv1.VolumeSnapshotContent) { +func (ctrl *csiSnapshotCommonController) updateContent(content *crdv1.VolumeSnapshotContent) { // Store the new content version in the cache and do not process it if this is // an old version. new, err := ctrl.storeContentUpdate(content) @@ -412,28 +394,31 @@ func (ctrl *csiSnapshotController) updateContent(content *crdv1.VolumeSnapshotCo } // deleteSnapshot runs in worker thread and handles "snapshot deleted" event. -func (ctrl *csiSnapshotController) deleteSnapshot(snapshot *crdv1.VolumeSnapshot) { +func (ctrl *csiSnapshotCommonController) deleteSnapshot(snapshot *crdv1.VolumeSnapshot) { _ = ctrl.snapshotStore.Delete(snapshot) - klog.V(4).Infof("snapshot %q deleted", snapshotKey(snapshot)) + klog.V(4).Infof("snapshot %q deleted", utils.SnapshotKey(snapshot)) - if snapshot.Status.BoundVolumeSnapshotContentName == nil { - klog.V(5).Infof("deleteSnapshot[%q]: content not bound", snapshotKey(snapshot)) + snapshotContentName := "" + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + if snapshotContentName == "" { + klog.V(5).Infof("deleteSnapshot[%q]: content not bound", utils.SnapshotKey(snapshot)) return } - snapshotContentName := *snapshot.Status.BoundVolumeSnapshotContentName // sync the content when its snapshot is deleted. Explicitly sync'ing the // content here in response to snapshot deletion prevents the content from // waiting until the next sync period for its Release. - klog.V(5).Infof("deleteSnapshot[%q]: scheduling sync of content %s", snapshotKey(snapshot), snapshotContentName) + klog.V(5).Infof("deleteSnapshot[%q]: scheduling sync of content %s", utils.SnapshotKey(snapshot), snapshotContentName) ctrl.contentQueue.Add(snapshotContentName) } // deleteContent runs in worker thread and handles "content deleted" event. -func (ctrl *csiSnapshotController) deleteContent(content *crdv1.VolumeSnapshotContent) { +func (ctrl *csiSnapshotCommonController) deleteContent(content *crdv1.VolumeSnapshotContent) { _ = ctrl.contentStore.Delete(content) klog.V(4).Infof("content %q deleted", content.Name) - snapshotName := snapshotRefKey(content.Spec.VolumeSnapshotRef) + snapshotName := utils.SnapshotRefKey(&content.Spec.VolumeSnapshotRef) if snapshotName == "" { klog.V(5).Infof("deleteContent[%q]: content not bound", content.Name) return @@ -448,7 +433,7 @@ func (ctrl *csiSnapshotController) deleteContent(content *crdv1.VolumeSnapshotCo // initializeCaches fills all controller caches with initial data from etcd in // order to have the caches already filled when first addSnapshot/addContent to // perform initial synchronization of the controller. -func (ctrl *csiSnapshotController) initializeCaches(snapshotLister storagelisters.VolumeSnapshotLister, contentLister storagelisters.VolumeSnapshotContentLister) { +func (ctrl *csiSnapshotCommonController) initializeCaches(snapshotLister storagelisters.VolumeSnapshotLister, contentLister storagelisters.VolumeSnapshotContentLister) { snapshotList, err := snapshotLister.List(labels.Everything()) if err != nil { klog.Errorf("CSISnapshotController can't initialize caches: %v", err) diff --git a/pkg/controller/snapshot_controller_test.go b/pkg/common-controller/snapshot_controller_test.go similarity index 93% rename from pkg/controller/snapshot_controller_test.go rename to pkg/common-controller/snapshot_controller_test.go index dc89bd253..e1785208f 100644 --- a/pkg/controller/snapshot_controller_test.go +++ b/pkg/common-controller/snapshot_controller_test.go @@ -14,21 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "testing" crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" "k8s.io/client-go/tools/cache" ) var deletionPolicy = crdv1.VolumeSnapshotContentDelete func storeVersion(t *testing.T, prefix string, c cache.Store, version string, expectedReturn bool) { - content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false) + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, true) content.ResourceVersion = version - ret, err := storeObjectUpdate(c, content, "content") + ret, err := utils.StoreObjectUpdate(c, content, "content") if err != nil { t.Errorf("%s: expected storeObjectUpdate to succeed, got: %v", prefix, err) } @@ -84,9 +85,9 @@ func TestControllerCacheParsingError(t *testing.T) { c := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc) // There must be something in the cache to compare with storeVersion(t, "Step1", c, "1", true) - content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false) + content := newContent("contentName", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "pv-handle-1-1", deletionPolicy, nil, nil, false, true) content.ResourceVersion = "xxx" - _, err := storeObjectUpdate(c, content, "content") + _, err := utils.StoreObjectUpdate(c, content, "content") if err == nil { t.Errorf("Expected parsing error, got nil instead") } diff --git a/pkg/controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go similarity index 63% rename from pkg/controller/snapshot_create_test.go rename to pkg/common-controller/snapshot_create_test.go index 2d691eccb..21602de8e 100644 --- a/pkg/controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" "time" @@ -71,12 +71,12 @@ func TestCreateSnapshotSync(t *testing.T) { { name: "6-1 - successful create snapshot with snapshot class gold", initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-1", "snapuid6-1", "snap6-1", "sid6-1", classGold, "", "pv-handle6-1", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &True, metaTimeNowUnix, getSize(defaultSize), nil), + expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-1", "snapuid6-1", "snap6-1", "sid6-1", classGold, "", "pv-handle6-1", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap6-1", "snapuid6-1", "claim6-1", "", classGold, "snapcontent-snapuid6-1", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-1", "pvc-uid6-1", "1Gi", "volume6-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-1", "pv-uid6-1", "pv-handle6-1", "1Gi", "pvc-uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid6-1", volume: newVolume("volume6-1", "pv-uid6-1", "pv-handle6-1", "1Gi", "pvc-uid6-1", "claim6-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -88,19 +88,19 @@ func TestCreateSnapshotSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, { name: "6-2 - successful create snapshot with snapshot class silver", initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &True, metaTimeNowUnix, getSize(defaultSize), nil), + expectedContents: newContentArrayNoStatus("snapcontent-snapuid6-2", "snapuid6-2", "snap6-2", "sid6-2", classSilver, "", "pv-handle6-2", deletionPolicy, nil, nil, false, false), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "snapcontent-snapuid6-2", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid6-2", volume: newVolume("volume6-2", "pv-uid6-2", "pv-handle6-2", "1Gi", "pvc-uid6-2", "claim6-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -112,56 +112,7 @@ func TestCreateSnapshotSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, - errors: noerrors, - test: testSyncSnapshot, - }, - { - name: "6-5 - successful create snapshot with status uploading", - initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-5", "snapuid6-5", "snap6-5", "sid6-5", classGold, "", "pv-handle6-5", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap6-5", "snapuid6-5", "claim6-5", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-5", "snapuid6-5", "claim6-5", "", classGold, "snapcontent-snapuid6-5", &False, metaTimeNowUnix, getSize(defaultSize), nil), - initialClaims: newClaimArray("claim6-5", "pvc-uid6-5", "1Gi", "volume6-5", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume6-5", "pv-uid6-5", "pv-handle6-5", "1Gi", "pvc-uid6-5", "claim6-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid6-5", - volume: newVolume("volume6-5", "pv-uid6-5", "pv-handle6-5", "1Gi", "pvc-uid6-5", "claim6-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid6-5", - creationTime: timeNow, - readyToUse: False, - }, - }, - errors: noerrors, - test: testSyncSnapshot, - }, - { - // TODO(xiangqian): this test does not match its name - name: "6-6 - successful create snapshot with status error uploading", - initialContents: nocontents, - expectedContents: newContentArrayWithReadyToUse("snapcontent-snapuid6-6", "snapuid6-6", "snap6-6", "sid6-6", classGold, "", "pv-handle6-6", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap6-6", "snapuid6-6", "claim6-6", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap6-6", "snapuid6-6", "claim6-6", "", classGold, "snapcontent-snapuid6-6", &False, metaTimeNowUnix, getSize(defaultSize), nil), - initialClaims: newClaimArray("claim6-6", "pvc-uid6-6", "1Gi", "volume6-6", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume6-6", "pv-uid6-6", "pv-handle6-6", "1Gi", "pvc-uid6-6", "claim6-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid6-6", - volume: newVolume("volume6-6", "pv-uid6-6", "pv-handle6-6", "1Gi", "pvc-uid6-6", "claim6-6", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid6-6", - creationTime: timeNow, - readyToUse: False, - }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, @@ -169,15 +120,16 @@ func TestCreateSnapshotSync(t *testing.T) { name: "7-1 - fail to create snapshot with non-existing snapshot class", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-1", "snapuid7-1", "claim7-1", "", classNonExisting, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-1: \"failed to retrieve snapshot class non-existing from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"non-existing\\\\\\\" not found\\\"\""), false), initialClaims: newClaimArray("claim7-1", "pvc-uid7-1", "1Gi", "volume7-1", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-1", "pv-uid7-1", "pv-handle7-1", "1Gi", "pvc-uid7-1", "claim7-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-2 - fail to create snapshot with snapshot class invalid-secret-class", initialContents: nocontents, expectedContents: nocontents, @@ -188,59 +140,63 @@ func TestCreateSnapshotSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotCreationFailed"}, errors: noerrors, test: testSyncSnapshot, - }, + },*/ { name: "7-3 - fail to create snapshot without snapshot class ", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-3: \"failed to retrieve snapshot class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-3", "snapuid7-3", "claim7-3", "", "", "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-3: \"failed to retrieve snapshot class from the informer: \\\"volumesnapshotclass.snapshot.storage.k8s.io \\\\\\\"\\\\\\\" not found\\\"\""), false), initialClaims: newClaimArray("claim7-3", "pvc-uid7-3", "1Gi", "volume7-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-3", "pv-uid7-3", "pv-handle7-3", "1Gi", "pvc-uid7-3", "claim7-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialStorageClasses: []*storage.StorageClass{diffDriverStorageClass}, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-4: \"failed to retrieve PVC claim7-4 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim7-4\\\\\\\" not found\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-4", "snapuid7-4", "claim7-4", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-4: \"failed to retrieve PVC claim7-4 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim7-4\\\\\\\" not found\\\"\""), false), initialVolumes: newVolumeArray("volume7-4", "pv-uid7-4", "pv-handle7-4", "1Gi", "pvc-uid7-4", "claim7-4", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-5 - fail create snapshot with no-existing volume", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\"")), + initialSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-5", "snapuid7-5", "claim7-5", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-5: \"failed to retrieve PV volume7-5 from the API server: \\\"cannot find volume volume7-5\\\"\""), false), initialClaims: newClaimArray("claim7-5", "pvc-uid7-5", "1Gi", "volume7-5", v1.ClaimBound, &classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, { name: "7-6 - fail create snapshot with claim that is not yet bound", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\"")), + initialSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap7-6", "snapuid7-6", "claim7-6", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to get input parameters to create snapshot snap7-6: \"the PVC claim7-6 is not yet bound to a PV, will not attempt to take a snapshot\""), false), initialClaims: newClaimArray("claim7-6", "pvc-uid7-6", "1Gi", "", v1.ClaimPending, &classEmpty), - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, errors: noerrors, + expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-7 - fail create snapshot due to csi driver error", initialContents: nocontents, expectedContents: nocontents, initialSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: failed to take snapshot of the volume, volume7-7: \"mock create snapshot error\"")), + expectedSnapshots: newSnapshotArray("snap7-7", "snapuid7-7", "claim7-7", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot content with error failed to take snapshot of the volume, volume7-7: \"mock create snapshot error\"")), initialClaims: newClaimArray("claim7-7", "pvc-uid7-7", "1Gi", "volume7-7", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume7-7", "pv-uid7-7", "pv-handle7-7", "1Gi", "pvc-uid7-7", "claim7-7", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), expectedCreateCalls: []createCall{ @@ -253,41 +209,44 @@ func TestCreateSnapshotSync(t *testing.T) { }, }, errors: noerrors, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectSuccess: false, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, test: testSyncSnapshot, - }, - { - name: "7-8 - fail create snapshot due to cannot update snapshot status", - initialContents: nocontents, - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: snapshot controller failed to update default/snap7-8 on API server: mock update error")), - initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid7-8", - volume: newVolume("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: map[string]string{"param1": "value1"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid7-8", - creationTime: timeNow, - readyToUse: True, - }, + },*/ + /*{ + name: "7-8 - fail create snapshot due to cannot update snapshot status", + initialContents: nocontents, + expectedContents: nocontents, + initialSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap7-8", "snapuid7-8", "claim7-8", "", classGold, "", &False, nil, nil, newVolumeError("Failed to create snapshot: snapshot controller failed to update default/snap7-8 on API server: mock update error")), + initialClaims: newClaimArray("claim7-8", "pvc-uid7-8", "1Gi", "volume7-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + /*expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid7-8", + volume: newVolume("volume7-8", "pv-uid7-8", "pv-handle7-8", "1Gi", "pvc-uid7-8", "claim7-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + parameters: map[string]string{"param1": "value1"}, + // information to return + driverName: mockDriverName, + size: defaultSize, + snapshotId: "sid7-8", + creationTime: timeNow, + readyToUse: True, }, - errors: []reactorError{ + },*/ + /*errors: []reactorError{ // Inject error to the forth client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. // All other calls will succeed. {"update", "volumesnapshots", errors.New("mock update error")}, {"update", "volumesnapshots", errors.New("mock update error")}, {"update", "volumesnapshots", errors.New("mock update error")}, }, - expectedEvents: []string{"Warning SnapshotCreationFailed"}, + expectedEvents: []string{"Warning SnapshotContentCreationFailed"}, + + expectSuccess: false, test: testSyncSnapshot, - }, - { + },*/ + /*{ // TODO(xiangqian): this test case needs to be revisited the scenario // of VolumeSnapshotContent saving failure. Since there will be no content object // in API server, it could potentially cause leaking issue @@ -330,7 +289,7 @@ func TestCreateSnapshotSync(t *testing.T) { initialSecrets: []*v1.Secret{}, // no initial secret created errors: noerrors, test: testSyncSnapshot, - }, + },*/ } runSyncTests(t, tests, snapshotClasses) } diff --git a/pkg/controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go similarity index 64% rename from pkg/controller/snapshot_delete_test.go rename to pkg/common-controller/snapshot_delete_test.go index f3c157bd3..f69cb5c86 100644 --- a/pkg/controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -14,13 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,18 +35,18 @@ var class2Parameters = map[string]string{ } var class3Parameters = map[string]string{ - "param3": "value3", - snapshotterSecretNameKey: "name", + "param3": "value3", + utils.SnapshotterSecretNameKey: "name", } var class4Parameters = map[string]string{ - snapshotterSecretNameKey: "emptysecret", - snapshotterSecretNamespaceKey: "default", + utils.SnapshotterSecretNameKey: "emptysecret", + utils.SnapshotterSecretNamespaceKey: "default", } var class5Parameters = map[string]string{ - snapshotterSecretNameKey: "secret", - snapshotterSecretNamespaceKey: "default", + utils.SnapshotterSecretNameKey: "secret", + utils.SnapshotterSecretNamespaceKey: "default", } var snapshotClasses = []*crdv1.VolumeSnapshotClass{ @@ -110,7 +111,7 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ }, ObjectMeta: metav1.ObjectMeta{ Name: defaultClass, - Annotations: map[string]string{IsDefaultSnapshotClassAnnotation: "true"}, + Annotations: map[string]string{utils.IsDefaultSnapshotClassAnnotation: "true"}, }, Driver: mockDriverName, DeletionPolicy: crdv1.VolumeSnapshotContentDelete, @@ -124,50 +125,50 @@ var snapshotClasses = []*crdv1.VolumeSnapshotClass{ func TestDeleteSync(t *testing.T) { tests := []controllerTest{ { - name: "1-1 - content with empty snapshot class is deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", - initialContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, - test: testSyncContent, + name: "1-1 - content with empty snapshot class is deleted if it is bound to a non-exist snapshot and also has a snapshot uid specified", + initialContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-1", "snapuid1-1", "snap1-1", "sid1-1", classGold, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid1-1", nil, nil}}, + test: testSyncContent, }, { - name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", - initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, - test: testSyncContent, + name: "2-1 - content with empty snapshot class will not be deleted if it is bound to a non-exist snapshot but it does not have a snapshot uid specified", + initialContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content2-1", "", "snap2-1", "sid2-1", "", "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid2-1", nil, nil}}, + test: testSyncContent, }, { - name: "1-2 - successful delete with snapshot class that has empty secret parameter", - initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{emptySecret()}, - expectedEvents: noevents, - errors: noerrors, - expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, - test: testSyncContent, + name: "1-2 - successful delete with snapshot class that has empty secret parameter", + initialContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-2", "sid1-2", "snap1-2", "sid1-2", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + initialSecrets: []*v1.Secret{emptySecret()}, + expectedEvents: noevents, + errors: noerrors, + //expectedDeleteCalls: []deleteCall{{"sid1-2", map[string]string{}, nil}}, + test: testSyncContent, }, { - name: "1-3 - successful delete with snapshot class that has valid secret parameter", - initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, - test: testSyncContent, + name: "1-3 - successful delete with snapshot class that has valid secret parameter", + initialContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-3", "sid1-3", "snap1-3", "sid1-3", validSecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-3", map[string]string{"foo": "bar"}, nil}}, + test: testSyncContent, }, /*{ name: "1-4 - fail delete with snapshot class that has invalid secret parameter", @@ -179,7 +180,7 @@ func TestDeleteSync(t *testing.T) { errors: noerrors, test: testSyncContent, },*/ - { + /*{ name: "1-5 - csi driver delete snapshot returns error", initialContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-5", "sid1-5", "snap1-5", "sid1-5", validSecretClass, "", "", deletionPolicy, nil, nil, true), @@ -190,15 +191,15 @@ func TestDeleteSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotDeleteError"}, errors: noerrors, test: testSyncContent, - }, - { + },*/ + /*{ name: "1-6 - api server delete content returns error", initialContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-6", "sid1-6", "snap1-6", "sid1-6", validSecretClass, "", "", deletionPolicy, nil, nil, true), initialSnapshots: nosnapshots, expectedSnapshots: nosnapshots, initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, + //expectedDeleteCalls: []deleteCall{{"sid1-6", map[string]string{"foo": "bar"}, nil}}, expectedEvents: []string{"Warning SnapshotContentObjectDeleteError"}, errors: []reactorError{ // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshotContents().Delete call. @@ -206,56 +207,56 @@ func TestDeleteSync(t *testing.T) { {"delete", "volumesnapshotcontents", errors.New("mock delete error")}, }, test: testSyncContent, - }, + },*/ { // delete success - snapshot that the content was pointing to was deleted, and another // with the same name created. - name: "1-7 - prebound content is deleted while the snapshot exists", - initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil), - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, - test: testSyncContent, + name: "1-7 - prebound content is deleted while the snapshot exists", + initialContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: newContentArray("content1-7", "sid1-7", "snap1-7", "sid1-7", emptySecretClass, "", "", deletionPolicy, nil, nil, true), + initialSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-7", "snapuid1-7-x", "claim1-7", "", validSecretClass, "", &False, nil, nil, nil, false), + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-7", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: testSyncContent, }, { // delete success(?) - content is deleted before doDelete() starts - name: "1-8 - content is deleted before deleting", - initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", validSecretClass, "", "", deletionPolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - initialSecrets: []*v1.Secret{secret()}, - expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, - expectedEvents: noevents, - errors: noerrors, - test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotController, reactor *snapshotReactor) { + name: "1-8 - content is deleted before deleting", + initialContents: newContentArray("content1-8", "sid1-8", "snap1-8", "sid1-8", validSecretClass, "", "", deletionPolicy, nil, nil, true), + expectedContents: nocontents, + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + initialSecrets: []*v1.Secret{secret()}, + //expectedDeleteCalls: []deleteCall{{"sid1-8", map[string]string{"foo": "bar"}, nil}}, + expectedEvents: noevents, + errors: noerrors, + test: wrapTestWithInjectedOperation(testSyncContent, func(ctrl *csiSnapshotCommonController, reactor *snapshotReactor) { // Delete the volume before delete operation starts reactor.lock.Lock() delete(reactor.contents, "content1-8") reactor.lock.Unlock() }), }, - { + /*{ name: "1-9 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", initialContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), expectedContents: newContentArray("content1-9", "snapuid1-9", "snap1-9", "sid1-9", validSecretClass, "", "", deletionPolicy, nil, nil, true), initialSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-9", "snapuid1-9", "claim1-9", "", validSecretClass, "content1-9", &True, nil, nil, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, - }, + },*/ { name: "1-10 - will not delete content with retain policy set which is bound to a snapshot incorrectly", initialContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), expectedContents: newContentArray("content1-10", "snapuid1-10-x", "snap1-10", "sid1-10", validSecretClass, "", "", retainPolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-10", "snapuid1-10", "claim1-10", "", validSecretClass, "content1-10", &False, nil, nil, nil, false), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -265,8 +266,8 @@ func TestDeleteSync(t *testing.T) { name: "1-11 - content will not be deleted if it is bound to a snapshot correctly, snapsht uid is not specified", initialContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), expectedContents: newContentArray("content1-11", "", "snap1-11", "sid1-11", validSecretClass, "", "", deletePolicy, nil, nil, true), - initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap1-11", "snapuid1-11", "claim1-11", "", validSecretClass, "content1-11", &False, nil, nil, nil, false), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, @@ -292,28 +293,28 @@ func TestDeleteSync(t *testing.T) { errors: noerrors, test: testSyncContent, }, - { + /*{ name: "1-14 - content will not be deleted if it is bound to a snapshot correctly, snapshot uid is specified", initialContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), expectedContents: newContentArray("content1-14", "snapuid1-14", "snap1-14", "sid1-14", validSecretClass, "", "", retainPolicy, nil, nil, true), initialSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &False, nil, nil, nil), + expectedSnapshots: newSnapshotArray("snap1-14", "snapuid1-14", "claim1-14", "", validSecretClass, "content1-14", &True, nil, nil, nil), expectedEvents: noevents, initialSecrets: []*v1.Secret{secret()}, errors: noerrors, test: testSyncContent, - }, + },*/ { - name: "1-16 - continue delete with snapshot class that has nonexistent secret", - initialContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), - expectedContents: nocontents, - initialSnapshots: nosnapshots, - expectedSnapshots: nosnapshots, - expectedEvents: noevents, - errors: noerrors, - initialSecrets: []*v1.Secret{}, // secret does not exist - expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, - test: testSyncContent, + name: "1-16 - continue delete with snapshot class that has nonexistent secret", + initialContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), + expectedContents: newContentArray("content1-16", "sid1-16", "snap1-16", "sid1-16", emptySecretClass, "", "", deletePolicy, nil, nil, true), + initialSnapshots: nosnapshots, + expectedSnapshots: nosnapshots, + expectedEvents: noevents, + errors: noerrors, + initialSecrets: []*v1.Secret{}, // secret does not exist + //expectedDeleteCalls: []deleteCall{{"sid1-16", nil, nil}}, + test: testSyncContent, }, } runSyncTests(t, tests, snapshotClasses) diff --git a/pkg/controller/snapshot_finalizer_test.go b/pkg/common-controller/snapshot_finalizer_test.go similarity index 85% rename from pkg/controller/snapshot_finalizer_test.go rename to pkg/common-controller/snapshot_finalizer_test.go index 296b78750..049262fdf 100644 --- a/pkg/controller/snapshot_finalizer_test.go +++ b/pkg/common-controller/snapshot_finalizer_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( "testing" @@ -22,42 +22,42 @@ import ( v1 "k8s.io/api/core/v1" ) -// Test single call to ensureSnapshotSourceFinalizer and checkandRemoveSnapshotSourceFinalizer, +// Test single call to ensurePVCFinalizer and checkandRemovePVCFinalizer, // expecting PVCFinalizer to be added or removed func TestPVCFinalizer(t *testing.T) { tests := []controllerTest{ { name: "1-1 - successful add PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: true, }, { name: "1-2 - won't add PVC finalizer; already added", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testAddPVCFinalizer, expectSuccess: false, }, { name: "1-3 - successful remove PVC finalizer", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArrayFinalizer("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: true, }, { name: "1-4 - won't remove PVC finalizer; already removed", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, }, { name: "1-5 - won't remove PVC finalizer; PVC in-use", - initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil), + initialSnapshots: newSnapshotArray("snap6-2", "snapuid6-2", "claim6-2", "", classSilver, "", &False, nil, nil, nil, false), initialClaims: newClaimArray("claim6-2", "pvc-uid6-2", "1Gi", "volume6-2", v1.ClaimBound, &classEmpty), test: testRemovePVCFinalizer, expectSuccess: false, diff --git a/pkg/controller/snapshot_ready_test.go b/pkg/common-controller/snapshot_update_test.go similarity index 68% rename from pkg/controller/snapshot_ready_test.go rename to pkg/common-controller/snapshot_update_test.go index e39b9f1f2..f3a2faea6 100644 --- a/pkg/controller/snapshot_ready_test.go +++ b/pkg/common-controller/snapshot_update_test.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package common_controller import ( - "errors" + //"errors" "testing" "time" @@ -41,19 +41,20 @@ var volumeErr = &storagev1beta1.VolumeError{ // controllerTest.testCall *once*. // 3. Compare resulting contents and snapshots with expected contents and snapshots. func TestSync(t *testing.T) { + size := int64(1) tests := []controllerTest{ { // snapshot is bound to a non-existing content name: "2-1 - snapshot is bound to a non-existing content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &True, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-1", "snapuid2-1", "claim2-1", "", validSecretClass, "content2-1", &False, nil, nil, newVolumeError("VolumeSnapshotContent is missing"), false), expectedEvents: []string{"Warning SnapshotContentMissing"}, errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "2-2 - snapshot points to a content but content does not point to snapshot(VolumeSnapshotRef does not match)", initialContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-2", "snapuid2-2-x", "snap2-2", "sid2-2", validSecretClass, "", "", deletionPolicy, nil, nil, false), @@ -62,17 +63,17 @@ func TestSync(t *testing.T) { expectedEvents: []string{"Warning InvalidSnapshotBinding"}, errors: noerrors, test: testSyncSnapshotError, - }, + },*/ { name: "2-3 - success bind snapshot and content but not ready, no status changed", initialContents: newContentArray("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), - expectedContents: newContentArrayWithReadyToUse("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &False, false), - initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-3", "snapuid2-3", "snap2-3", "sid2-3", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &True, false), + initialSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-3", "snapuid2-3", "claim2-3", "", validSecretClass, "content2-3", &True, metaTimeNow, nil, nil, false), initialClaims: newClaimArray("claim2-3", "pvc-uid2-3", "1Gi", "volume2-3", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid2-3", volume: newVolume("volume2-3", "pv-uid2-3", "pv-handle2-3", "1Gi", "pvc-uid2-3", "claim2-3", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -85,7 +86,7 @@ func TestSync(t *testing.T) { creationTime: timeNow, readyToUse: False, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, @@ -94,21 +95,21 @@ func TestSync(t *testing.T) { name: "2-4 - noop", initialContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-4", "snapuid2-4", "snap2-4", "sid2-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil), + initialSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-4", "snapuid2-4", "claim2-4", "", validSecretClass, "content2-4", &True, metaTimeNow, nil, nil, false), errors: noerrors, test: testSyncSnapshot, }, { name: "2-5 - snapshot and content bound, status ready false -> true", initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, metaTimeNow, nil, nil, false), initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ + /*expectedCreateCalls: []createCall{ { snapshotName: "snapshot-snapuid2-5", volume: newVolume("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), @@ -121,16 +122,16 @@ func TestSync(t *testing.T) { creationTime: timeNow, readyToUse: True, }, - }, + },*/ errors: noerrors, test: testSyncSnapshot, }, { name: "2-6 - snapshot bound to prebound content correctly, status ready false -> true, ref.UID '' -> 'snapuid2-6'", initialContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, &defaultSize, &True, false), - initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &True, metaTimeNow, getSize(defaultSize), nil), + expectedContents: newContentArrayWithReadyToUse("content2-6", "snapuid2-6", "snap2-6", "sid2-6", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-6", "snapuid2-6", "", "content2-6", validSecretClass, "content2-6", &False, metaTimeNow, nil, nil, false), expectedListCalls: []listCall{ { size: defaultSize, @@ -141,7 +142,7 @@ func TestSync(t *testing.T) { errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "2-7 - snapshot and content bound, csi driver get status error", initialContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), expectedContents: newContentArrayWithReadyToUse("content2-7", "snapuid2-7", "snap2-7", "sid2-7", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), @@ -163,53 +164,54 @@ func TestSync(t *testing.T) { }, errors: noerrors, test: testSyncSnapshot, - }, - { - name: "2-8 - snapshot and content bound, apiserver update status error", - initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), - initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: snapshot controller failed to update default/snap2-8 on API server: mock update error")), - expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, - initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), - initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - initialSecrets: []*v1.Secret{secret()}, - expectedCreateCalls: []createCall{ - { - snapshotName: "snapshot-snapuid2-8", - volume: newVolume("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), - parameters: class5Parameters, - secrets: map[string]string{"foo": "bar"}, - // information to return - driverName: mockDriverName, - size: defaultSize, - snapshotId: "sid2-8", - creationTime: timeNow, - readyToUse: true, - }, + },*/ + /*{ + name: "2-8 - snapshot and content bound, apiserver update status error", + initialContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + expectedContents: newContentArrayWithReadyToUse("content2-8", "snapuid2-8", "snap2-8", "sid2-8", validSecretClass, "", "", deletionPolicy, &timeNowStamp, nil, &False, false), + initialSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, nil), + expectedSnapshots: newSnapshotArray("snap2-8", "snapuid2-8", "claim2-8", "", validSecretClass, "content2-8", &False, metaTimeNow, nil, newVolumeError("Failed to check and update snapshot: snapshot controller failed to update default/snap2-8 on API server: mock update error")), + expectedEvents: []string{"Warning SnapshotCheckandUpdateFailed"}, + initialClaims: newClaimArray("claim2-8", "pvc-uid2-8", "1Gi", "volume2-8", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + /*expectedCreateCalls: []createCall{ + { + snapshotName: "snapshot-snapuid2-8", + volume: newVolume("volume2-8", "pv-uid2-8", "pv-handle2-8", "1Gi", "pvc-uid2-8", "claim2-8", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + parameters: class5Parameters, + secrets: map[string]string{"foo": "bar"}, + // information to return + driverName: mockDriverName, + size: defaultSize, + snapshotId: "sid2-8", + creationTime: timeNow, + readyToUse: true, }, + },*/ /* errors: []reactorError{ // Inject error to the first client.VolumesnapshotV1beta1().VolumeSnapshots().Update call. // All other calls will succeed. {"update", "volumesnapshots", errors.New("mock update error")}, }, test: testSyncSnapshot, - }, + },*/ { name: "2-9 - fail on status update as there is not pvc provided", initialContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-9", "snapuid2-9", "snap2-9", "sid2-9", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil), - expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), - errors: noerrors, - test: testSyncSnapshot, + initialSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &True, nil, nil, nil, false), + //expectedSnapshots: newSnapshotArray("snap2-9", "snapuid2-9", "claim2-9", "", validSecretClass, "content2-9", &False, nil, nil, newVolumeError("Failed to check and update snapshot: failed to get input parameters to create snapshot snap2-9: \"failed to retrieve PVC claim2-9 from the lister: \\\"persistentvolumeclaim \\\\\\\"claim2-9\\\\\\\" not found\\\"\"")), + errors: noerrors, + test: testSyncSnapshot, }, { name: "2-10 - do not bind when snapshot and content not match", initialContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content2-10", "snapuid2-10-x", "snap2-10", "sid2-10", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error")), - expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error")), + initialSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false), + expectedSnapshots: newSnapshotArray("snap2-10", "snapuid2-10", "claim2-10", "", validSecretClass, "", &False, nil, nil, newVolumeError("mock driver error"), false), errors: noerrors, test: testSyncSnapshot, }, @@ -217,8 +219,8 @@ func TestSync(t *testing.T) { name: "3-1 - ready snapshot lost reference to VolumeSnapshotContent", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-1", "snapuid3-1", "claim3-1", "", validSecretClass, "content3-1", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -227,8 +229,8 @@ func TestSync(t *testing.T) { name: "3-2 - ready snapshot bound to none-exist content", initialContents: nocontents, expectedContents: nocontents, - initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing")), + initialSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-2", "snapuid3-2", "claim3-2", "", validSecretClass, "content3-2", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is missing"), false), errors: noerrors, expectedEvents: []string{"Warning SnapshotContentMissing"}, test: testSyncSnapshot, @@ -237,8 +239,8 @@ func TestSync(t *testing.T) { name: "3-3 - ready snapshot(everything is well, do nothing)", initialContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-3", "snapuid3-3", "snap3-3", "sid3-3", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil), + initialSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-3", "snapuid3-3", "claim3-3", "", validSecretClass, "content3-3", &True, metaTimeNow, nil, nil, false), errors: noerrors, test: testSyncSnapshot, }, @@ -246,12 +248,12 @@ func TestSync(t *testing.T) { name: "3-4 - ready snapshot misbound to VolumeSnapshotContent", initialContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: newContentArray("content3-4", "snapuid3-4-x", "snap3-4", "sid3-4", validSecretClass, "", "", deletionPolicy, nil, nil, false), - initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil), - expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly")), + initialSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &True, metaTimeNow, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap3-4", "snapuid3-4", "claim3-4", "", validSecretClass, "content3-4", &False, metaTimeNow, nil, newVolumeError("VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"), false), errors: noerrors, test: testSyncSnapshot, }, - { + /*{ name: "3-5 - snapshot bound to content in which the driver does not match", initialContents: newContentWithUnmatchDriverArray("content3-5", "snapuid3-5", "snap3-5", "sid3-5", validSecretClass, "", "", deletionPolicy, nil, nil, false), expectedContents: nocontents, @@ -260,6 +262,42 @@ func TestSync(t *testing.T) { expectedEvents: []string{"Warning SnapshotContentMissing"}, errors: noerrors, test: testSyncSnapshotError, + },*/ + { + name: "4-1 - content bound to snapshot, snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, true), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, + { + name: "4-2 - snapshot and content bound, ReadyToUse in snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, nil, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, nil, nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, + }, + { + name: "4-3 - content bound to snapshot, fields in snapshot status missing and rebuilt", + initialContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + expectedContents: newContentArrayWithReadyToUse("content2-5", "snapuid2-5", "snap2-5", "sid2-5", validSecretClass, "", "", deletionPolicy, nil, &size, &True, false), + initialSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "", &False, nil, nil, nil, false), + expectedSnapshots: newSnapshotArray("snap2-5", "snapuid2-5", "claim2-5", "", validSecretClass, "content2-5", &True, nil, getSize(1), nil, false), + initialClaims: newClaimArray("claim2-5", "pvc-uid2-5", "1Gi", "volume2-5", v1.ClaimBound, &classEmpty), + initialVolumes: newVolumeArray("volume2-5", "pv-uid2-5", "pv-handle2-5", "1Gi", "pvc-uid2-5", "claim2-5", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, classEmpty), + initialSecrets: []*v1.Secret{secret()}, + errors: noerrors, + test: testSyncSnapshot, }, } diff --git a/pkg/controller/snapshot_controller.go b/pkg/controller/snapshot_controller.go deleted file mode 100644 index 88f6ed206..000000000 --- a/pkg/controller/snapshot_controller.go +++ /dev/null @@ -1,1226 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "fmt" - "strings" - "time" - - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - v1 "k8s.io/api/core/v1" - storagev1 "k8s.io/api/storage/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes/scheme" - ref "k8s.io/client-go/tools/reference" - "k8s.io/klog" - "k8s.io/kubernetes/pkg/util/goroutinemap" - "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" - "k8s.io/kubernetes/pkg/util/slice" -) - -// ================================================================== -// PLEASE DO NOT ATTEMPT TO SIMPLIFY THIS CODE. -// KEEP THE SPACE SHUTTLE FLYING. -// ================================================================== - -// Design: -// -// The fundamental key to this design is the bi-directional "pointer" between -// VolumeSnapshots and VolumeSnapshotContents, which is represented here -// as snapshot.Spec.Source.VolumeSnapshotContentName and content.Spec.VolumeSnapshotRef. -// The bi-directionality is complicated to manage in a transactionless system, but -// without it we can't ensure sane behavior in the face of different forms of -// trouble. For example, a rogue HA controller instance could end up racing -// and making multiple bindings that are indistinguishable, resulting in -// potential data loss. -// -// This controller is designed to work in active-passive high availability -// mode. It *could* work also in active-active HA mode, all the object -// transitions are designed to cope with this, however performance could be -// lower as these two active controllers will step on each other toes -// frequently. -// -// This controller supports both dynamic snapshot creation and pre-bound snapshot. -// In pre-bound mode, objects are created with pre-defined pointers: a VolumeSnapshot -// points to a specific VolumeSnapshotContent and the VolumeSnapshotContent also -// points back for this VolumeSnapshot. -// -// The dynamic snapshot creation is multi-step process: first controller triggers -// snapshot creation though csi volume plugin which should return a snapshot after -// it is created successfully (however, the snapshot might not be ready to use yet if -// there is an uploading phase). The creationTimestamp will be updated according to -// VolumeSnapshot, and then a VolumeSnapshotContent object is created to represent -// this snapshot. After that, the controller will keep checking the snapshot status -// though csi snapshot calls. When the snapshot is ready to use, the controller set -// the status "Bound" to true to indicate the snapshot is bound and ready to use. -// If the createtion failed for any reason, the Error status is set accordingly. -// In alpha version, the controller not retry to create the snapshot after it failed. -// In the future version, a retry policy will be added. - -const pvcKind = "PersistentVolumeClaim" -const apiGroup = "" -const snapshotKind = "VolumeSnapshot" -const snapshotAPIGroup = crdv1.GroupName - -const controllerUpdateFailMsg = "snapshot controller failed to update" - -const IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" - -// syncContent deals with one key off the queue. It returns false when it's time to quit. -func (ctrl *csiSnapshotController) syncContent(content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) - - if isContentDeletionCandidate(content) { - // Volume snapshot content should be deleted. Check if it's used - // and remove finalizer if it's not. - // Check if snapshot content is still bound to a snapshot. - isUsed := ctrl.isSnapshotContentBeingUsed(content) - if !isUsed { - klog.V(5).Infof("syncContent: Remove Finalizer for VolumeSnapshotContent[%s]", content.Name) - return ctrl.removeContentFinalizer(content) - } - } - - if needToAddContentFinalizer(content) { - // Content is not being deleted -> it should have the finalizer. - klog.V(5).Infof("syncContent: Add Finalizer for VolumeSnapshotContent[%s]", content.Name) - return ctrl.addContentFinalizer(content) - } - - // VolumeSnapshotContent is not bound to any VolumeSnapshot, in this case we just return err - if content.Spec.VolumeSnapshotRef.Name == "" { - // content is not bound - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is not bound to any VolumeSnapshot", content.Name) - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotContentNotBound", "VolumeSnapshotContent is not bound to any VolumeSnapshot") - return fmt.Errorf("volumeSnapshotContent %s is not bound to any VolumeSnapshot", content.Name) - } - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content is bound to snapshot %s", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // The VolumeSnapshotContent is reserved for a VolumeSnapshot; - // that VolumeSnapshot has not yet been bound to this VolumeSnapshotContent; the VolumeSnapshot sync will handle it. - if content.Spec.VolumeSnapshotRef.UID == "" { - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: VolumeSnapshotContent is pre-bound to VolumeSnapshot %s", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - return nil - } - // Get the VolumeSnapshot by _name_ - var snapshot *crdv1.VolumeSnapshot - snapshotName := snapshotRefKey(content.Spec.VolumeSnapshotRef) - obj, found, err := ctrl.snapshotStore.GetByKey(snapshotName) - if err != nil { - return err - } - if !found { - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s not found", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // Fall through with snapshot = nil - } else { - var ok bool - snapshot, ok = obj.(*crdv1.VolumeSnapshot) - if !ok { - return fmt.Errorf("cannot convert object from snapshot cache to snapshot %q!?: %#v", content.Name, obj) - } - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: snapshot %s found", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - } - if snapshot != nil && snapshot.UID != content.Spec.VolumeSnapshotRef.UID { - // The snapshot that the content was pointing to was deleted, and another - // with the same name created. - klog.V(4).Infof("synchronizing VolumeSnapshotContent[%s]: content %s has different UID, the old one must have been deleted", content.Name, snapshotRefKey(content.Spec.VolumeSnapshotRef)) - // Treat the content as bound to a missing snapshot. - snapshot = nil - } - if snapshot == nil { - switch content.Spec.DeletionPolicy { - case crdv1.VolumeSnapshotContentRetain: - klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Retain, nothing to do", content.Name) - - case crdv1.VolumeSnapshotContentDelete: - klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Delete", content.Name) - ctrl.deleteSnapshotContent(content) - default: - // Unknown VolumeSnapshotDeletionolicy - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotUnknownDeletionPolicy", "Volume Snapshot Content has unrecognized deletion policy") - } - return nil - } - return nil -} - -// syncSnapshot is the main controller method to decide what to do with a snapshot. -// It's invoked by appropriate cache.Controller callbacks when a snapshot is -// created, updated or periodically synced. We do not differentiate between -// these events. -// For easier readability, it is split into syncUnreadySnapshot and syncReadySnapshot -func (ctrl *csiSnapshotController) syncSnapshot(snapshot *crdv1.VolumeSnapshot) error { - klog.V(5).Infof("synchonizing VolumeSnapshot[%s]: %s", snapshotKey(snapshot), getSnapshotStatusForLogging(snapshot)) - - if isSnapshotDeletionCandidate(snapshot) { - // Volume snapshot should be deleted. Check if it's used - // and remove finalizer if it's not. - // Check if a volume is being created from snapshot. - isUsed := ctrl.isVolumeBeingCreatedFromSnapshot(snapshot) - if !isUsed { - klog.V(5).Infof("syncSnapshot: Remove Finalizer for VolumeSnapshot[%s]", snapshotKey(snapshot)) - return ctrl.removeSnapshotFinalizer(snapshot) - } - } - - if needToAddSnapshotFinalizer(snapshot) { - // Snapshot is not being deleted -> it should have the finalizer. - klog.V(5).Infof("syncSnapshot: Add Finalizer for VolumeSnapshot[%s]", snapshotKey(snapshot)) - return ctrl.addSnapshotFinalizer(snapshot) - } - - klog.V(5).Infof("syncSnapshot[%s]: check if we should remove finalizer on snapshot source and remove it if we can", snapshotKey(snapshot)) - // Check if we should remove finalizer on snapshot source and remove it if we can. - errFinalizer := ctrl.checkandRemoveSnapshotSourceFinalizer(snapshot) - if errFinalizer != nil { - klog.Errorf("error check and remove snapshot source finalizer for snapshot [%s]: %v", snapshot.Name, errFinalizer) - // Log an event and keep the original error from syncUnready/ReadySnapshot - ctrl.eventRecorder.Event(snapshot, v1.EventTypeWarning, "ErrorSnapshotSourceFinalizer", "Error check and remove PVC Finalizer for VolumeSnapshot") - } - - if !isSnapshotReadyToUse(snapshot) { - return ctrl.syncUnreadySnapshot(snapshot) - } - return ctrl.syncReadySnapshot(snapshot) -} - -// syncReadySnapshot checks the snapshot which has been bound to snapshot content successfully before. -// If there is any problem with the binding (e.g., snapshot points to a non-exist snapshot content), update the snapshot status and emit event. -func (ctrl *csiSnapshotController) syncReadySnapshot(snapshot *crdv1.VolumeSnapshot) error { - if snapshot.Status.BoundVolumeSnapshotContentName == nil { - if err := ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotLost", "Bound snapshot has lost reference to VolumeSnapshotContent"); err != nil { - return err - } - return nil - } - obj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing"); err != nil { - return err - } - return nil - } else { - content, ok := obj.(*crdv1.VolumeSnapshotContent) - if !ok { - return fmt.Errorf("Cannot convert object from snapshot content store to VolumeSnapshotContent %q!?: %#v", *snapshot.Status.BoundVolumeSnapshotContentName, obj) - } - - klog.V(5).Infof("syncReadySnapshot[%s]: VolumeSnapshotContent %q found", snapshotKey(snapshot), content.Name) - if !IsSnapshotBound(snapshot, content) { - // snapshot is bound but content is not bound to snapshot correctly - if err = ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotMisbound", "VolumeSnapshotContent is not bound to the VolumeSnapshot correctly"); err != nil { - return err - } - return nil - } - // Snapshot is correctly bound. - return nil - } -} - -// syncUnreadySnapshot is the main controller method to decide what to do with a snapshot which is not set to ready. -func (ctrl *csiSnapshotController) syncUnreadySnapshot(snapshot *crdv1.VolumeSnapshot) error { - uniqueSnapshotName := snapshotKey(snapshot) - klog.V(5).Infof("syncUnreadySnapshot %s", uniqueSnapshotName) - - if snapshot.Spec.Source.VolumeSnapshotContentName != nil { - contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Spec.Source.VolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - // snapshot is bound to a non-existing content. - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") - klog.V(4).Infof("synchronizing unready snapshot[%s]: snapshotcontent %q requested and not found, will try again next time", uniqueSnapshotName, *snapshot.Spec.Source.VolumeSnapshotContentName) - return fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, *snapshot.Spec.Source.VolumeSnapshotContentName) - } - content, ok := contentObj.(*crdv1.VolumeSnapshotContent) - if !ok { - return fmt.Errorf("expected volume snapshot content, got %+v", contentObj) - } - contentBound, err := ctrl.checkandBindSnapshotContent(snapshot, content) - if err != nil { - // snapshot is bound but content is not bound to snapshot correctly - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotBindFailed", fmt.Sprintf("Snapshot failed to bind VolumeSnapshotContent, %v", err)) - return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly, %v", uniqueSnapshotName, content.Name, err) - } - // snapshot is already bound correctly, check the status and update if it is ready. - klog.V(5).Infof("Check and update snapshot %s status", uniqueSnapshotName) - if err = ctrl.checkandUpdateBoundSnapshotStatus(snapshot, contentBound); err != nil { - return err - } - return nil - } else { // snapshot.Source.Spec.VolumeSnapshotContentName == nil - // find a matching volume snapshot content - if contentObj := ctrl.getMatchSnapshotContent(snapshot); contentObj != nil { - klog.V(5).Infof("Find VolumeSnapshotContent object %s for snapshot %s", contentObj.Name, uniqueSnapshotName) - if err := ctrl.checkandUpdateBoundSnapshotStatus(snapshot, contentObj); err != nil { - return err - } - klog.V(5).Infof("checkandUpdateBoundSnapshotStatus %v", snapshot) - } else if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { - contentObj, found, err := ctrl.contentStore.GetByKey(*snapshot.Status.BoundVolumeSnapshotContentName) - if err != nil { - return err - } - if !found { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotContentMissing", "VolumeSnapshotContent is missing") - return fmt.Errorf("snapshot %s is bound to a non-existing content %s", uniqueSnapshotName, *snapshot.Status.BoundVolumeSnapshotContentName) - } else { - content, _ := contentObj.(*crdv1.VolumeSnapshotContent) - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "InvalidSnapshotBinding", fmt.Sprintf("Snapshot is bound to a VolumeSnapshotContent which is bound to other Snapshot")) - return fmt.Errorf("snapshot %s is bound, but VolumeSnapshotContent %s is not bound to the VolumeSnapshot correctly", uniqueSnapshotName, content.Name) - } - } else if snapshot.Status == nil || snapshot.Status.Error == nil || isControllerUpdateFailError(snapshot.Status.Error) { - if err := ctrl.createSnapshot(snapshot); err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err)) - return err - } - } - return nil - } -} - -// getMatchSnapshotContent looks up VolumeSnapshotContent for a VolumeSnapshot named snapshotName -func (ctrl *csiSnapshotController) getMatchSnapshotContent(snapshot *crdv1.VolumeSnapshot) *crdv1.VolumeSnapshotContent { - var snapshotContentObj *crdv1.VolumeSnapshotContent - var found bool - - objs := ctrl.contentStore.List() - for _, obj := range objs { - content := obj.(*crdv1.VolumeSnapshotContent) - if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && - content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && - content.Spec.VolumeSnapshotRef.UID == snapshot.UID { - found = true - snapshotContentObj = content - break - } - } - - if !found { - klog.V(4).Infof("No VolumeSnapshotContent for VolumeSnapshot %s found", snapshotKey(snapshot)) - return nil - } - - return snapshotContentObj -} - -// deleteSnapshotContent starts delete action. -func (ctrl *csiSnapshotController) deleteSnapshotContent(content *crdv1.VolumeSnapshotContent) { - operationName := fmt.Sprintf("delete-%s[%s]", content.Name, string(content.UID)) - klog.V(5).Infof("Snapshotter is about to delete volume snapshot content and the operation named %s", operationName) - ctrl.scheduleOperation(operationName, func() error { - return ctrl.deleteSnapshotContentOperation(content) - }) -} - -// scheduleOperation starts given asynchronous operation on given volume. It -// makes sure the operation is already not running. -func (ctrl *csiSnapshotController) scheduleOperation(operationName string, operation func() error) { - klog.V(5).Infof("scheduleOperation[%s]", operationName) - - err := ctrl.runningOperations.Run(operationName, operation) - if err != nil { - switch { - case goroutinemap.IsAlreadyExists(err): - klog.V(4).Infof("operation %q is already running, skipping", operationName) - case exponentialbackoff.IsExponentialBackoff(err): - klog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) - default: - klog.Errorf("error scheduling operation %q: %v", operationName, err) - } - } -} - -func (ctrl *csiSnapshotController) storeSnapshotUpdate(snapshot interface{}) (bool, error) { - return storeObjectUpdate(ctrl.snapshotStore, snapshot, "snapshot") -} - -func (ctrl *csiSnapshotController) storeContentUpdate(content interface{}) (bool, error) { - return storeObjectUpdate(ctrl.contentStore, content, "content") -} - -// createSnapshot starts new asynchronous operation to create snapshot -func (ctrl *csiSnapshotController) createSnapshot(snapshot *crdv1.VolumeSnapshot) error { - klog.V(5).Infof("createSnapshot[%s]: started", snapshotKey(snapshot)) - opName := fmt.Sprintf("create-%s[%s]", snapshotKey(snapshot), string(snapshot.UID)) - ctrl.scheduleOperation(opName, func() error { - snapshotObj, err := ctrl.createSnapshotOperation(snapshot) - if err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) - klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err) - return err - } - _, updateErr := ctrl.storeSnapshotUpdate(snapshotObj) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("createSnapshot [%s]: cannot update internal cache: %v", snapshotKey(snapshotObj), updateErr) - } - return nil - }) - return nil -} - -func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatus(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("checkandUpdateSnapshotStatus[%s] started", snapshotKey(snapshot)) - opName := fmt.Sprintf("check-%s[%s]", snapshotKey(snapshot), string(snapshot.UID)) - ctrl.scheduleOperation(opName, func() error { - snapshotObj, err := ctrl.checkandUpdateBoundSnapshotStatusOperation(snapshot, content) - if err != nil { - ctrl.updateSnapshotErrorStatusWithEvent(snapshot, v1.EventTypeWarning, "SnapshotCheckandUpdateFailed", fmt.Sprintf("Failed to check and update snapshot: %v", err)) - klog.Errorf("checkandUpdateSnapshotStatus [%s]: error occured %v", snapshotKey(snapshot), err) - return err - } - _, updateErr := ctrl.storeSnapshotUpdate(snapshotObj) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("checkandUpdateSnapshotStatus [%s]: cannot update internal cache: %v", snapshotKey(snapshotObj), updateErr) - } - - return nil - }) - return nil -} - -// updateSnapshotStatusWithEvent saves new snapshot.Status to API server and emits -// given event on the snapshot. It saves the status and emits the event only when -// the status has actually changed from the version saved in API server. -// Parameters: -// snapshot - snapshot to update -// eventtype, reason, message - event to send, see EventRecorder.Event() -func (ctrl *csiSnapshotController) updateSnapshotErrorStatusWithEvent(snapshot *crdv1.VolumeSnapshot, eventtype, reason, message string) error { - klog.V(5).Infof("updateSnapshotStatusWithEvent[%s]", snapshotKey(snapshot)) - - if snapshot.Status != nil && snapshot.Status.Error != nil && snapshot.Status.Error.Message != nil && *snapshot.Status.Error.Message == message { - klog.V(4).Infof("updateSnapshotStatusWithEvent[%s]: the same error %v is already set", snapshot.Name, snapshot.Status.Error) - return nil - } - snapshotClone := snapshot.DeepCopy() - statusError := &crdv1.VolumeSnapshotError{ - Time: &metav1.Time{ - Time: time.Now(), - }, - Message: &message, - } - if snapshotClone.Status == nil { - snapshotClone.Status = &crdv1.VolumeSnapshotStatus{} - } - snapshotClone.Status.Error = statusError - ready := false - snapshotClone.Status.ReadyToUse = &ready - newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) - - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] error status failed %v", snapshotKey(snapshot), err) - return err - } - - _, err = ctrl.storeSnapshotUpdate(newSnapshot) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", snapshotKey(snapshot), err) - return err - } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) - - return nil -} - -// Stateless functions -func getSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string { - snapshotContentName := "" - readyToUse := isSnapshotReadyToUse(snapshot) - if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { - snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName - } - return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, readyToUse) -} - -// IsSnapshotBound returns true/false if snapshot is bound -func IsSnapshotBound(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { - if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && - content.Spec.VolumeSnapshotRef.UID == snapshot.UID { - return true - } - return false -} - -// isSnapshotConentBeingUsed checks if snapshot content is bound to snapshot. -func (ctrl *csiSnapshotController) isSnapshotContentBeingUsed(content *crdv1.VolumeSnapshotContent) bool { - snapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(content.Spec.VolumeSnapshotRef.Namespace).Get(content.Spec.VolumeSnapshotRef.Name, metav1.GetOptions{}) - if err != nil { - klog.Infof("isSnapshotContentBeingUsed: Cannot get snapshot %s from api server: [%v]. VolumeSnapshot object may be deleted already.", content.Spec.VolumeSnapshotRef.Name, err) - return false - } - - // Check if the snapshot content is bound to the snapshot - if IsSnapshotBound(snapshotObj, content) && snapshotObj.Status != nil && *snapshotObj.Status.BoundVolumeSnapshotContentName == content.Name { - klog.Infof("isSnapshotContentBeingUsed: VolumeSnapshot %s is bound to volumeSnapshotContent [%s]", snapshotObj.Name, content.Name) - return true - } - - klog.V(5).Infof("isSnapshotContentBeingUsed: Snapshot content %s is not being used", content.Name) - return false -} - -// isVolumeBeingCreatedFromSnapshot checks if an volume is being created from the snapshot. -func (ctrl *csiSnapshotController) isVolumeBeingCreatedFromSnapshot(snapshot *crdv1.VolumeSnapshot) bool { - pvcList, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).List(labels.Everything()) - if err != nil { - klog.Errorf("Failed to retrieve PVCs from the lister to check if volume snapshot %s is being used by a volume: %q", snapshotKey(snapshot), err) - return false - } - for _, pvc := range pvcList { - if pvc.Spec.DataSource != nil && len(pvc.Spec.DataSource.Name) > 0 && pvc.Spec.DataSource.Name == snapshot.Name { - if pvc.Spec.DataSource.Kind == snapshotKind && *(pvc.Spec.DataSource.APIGroup) == snapshotAPIGroup { - if pvc.Status.Phase == v1.ClaimPending { - // A volume is being created from the snapshot - klog.Infof("isVolumeBeingCreatedFromSnapshot: volume %s is being created from snapshot %s", pvc.Name, pvc.Spec.DataSource.Name) - return true - } - } - } - } - klog.V(5).Infof("isVolumeBeingCreatedFromSnapshot: no volume is being created from snapshot %s", snapshotKey(snapshot)) - return false -} - -// The function checks whether the volumeSnapshotRef in snapshot content matches the given snapshot. If match, it binds the content with the snapshot -func (ctrl *csiSnapshotController) checkandBindSnapshotContent(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { - if content.Spec.VolumeSnapshotRef.Name != snapshot.Name { - return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) - } else if content.Spec.VolumeSnapshotRef.UID != "" && content.Spec.VolumeSnapshotRef.UID != snapshot.UID { - return nil, fmt.Errorf("Could not bind snapshot %s and content %s, the VolumeSnapshotRef does not match", snapshot.Name, content.Name) - } else if content.Spec.VolumeSnapshotRef.UID != "" { - return content, nil - } - contentClone := content.DeepCopy() - contentClone.Spec.VolumeSnapshotRef.UID = snapshot.UID - newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", newContent.Name, err) - return nil, err - } - _, err = ctrl.storeContentUpdate(newContent) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", newContent.Name, err) - return nil, err - } - return newContent, nil -} - -func (ctrl *csiSnapshotController) getCreateSnapshotInput(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *v1.PersistentVolume, string, *v1.SecretReference, error) { - className := snapshot.Spec.VolumeSnapshotClassName - var class *crdv1.VolumeSnapshotClass - var err error - if className != nil { - class, err = ctrl.GetSnapshotClass(*className) - if err != nil { - klog.Errorf("getCreateSnapshotInput failed to getClassFromVolumeSnapshot %s", err) - return nil, nil, "", nil, err - } - } else { - klog.Errorf("failed to getCreateSnapshotInput %s without a snapshot class", snapshot.Name) - return nil, nil, "", nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", snapshot.Name) - } - - volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) - if err != nil { - klog.Errorf("getCreateSnapshotInput failed to get PersistentVolume object [%s]: Error: [%#v]", snapshot.Name, err) - return nil, nil, "", nil, err - } - - // Create VolumeSnapshotContent name - contentName := GetSnapshotContentNameForSnapshot(snapshot) - - // Resolve snapshotting secret credentials. - snapshotterSecretRef, err := getSecretReference(class.Parameters, contentName, snapshot) - if err != nil { - return nil, nil, "", nil, err - } - - return class, volume, contentName, snapshotterSecretRef, nil -} - -func (ctrl *csiSnapshotController) checkandUpdateBoundSnapshotStatusOperation(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshot, error) { - var err error - var creationTime time.Time - var size int64 - var readyToUse = false - var driverName string - var snapshotID string - - if snapshot.Spec.Source.VolumeSnapshotContentName != nil { - klog.V(5).Infof("checkandUpdateBoundSnapshotStatusOperation: checking whether snapshot [%s] is pre-bound to content [%s]", snapshot.Name, content.Name) - readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content) - if err != nil { - klog.Errorf("checkandUpdateBoundSnapshotStatusOperation: failed to call get snapshot status to check whether snapshot is ready to use %q", err) - return nil, err - } - driverName, snapshotID = content.Spec.Driver, *content.Status.SnapshotHandle - } else { - class, volume, _, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) - if err != nil { - return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) - } - snapshotterCredentials, err := getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return nil, err - } - driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials) - if err != nil { - klog.Errorf("checkandUpdateBoundSnapshotStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err) - return nil, err - } - } - klog.V(5).Infof("checkandUpdateBoundSnapshotStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - - if creationTime.IsZero() { - creationTime = time.Now() - } - newSnapshot, err := ctrl.updateSnapshotStatus(snapshot, content.Name, readyToUse, creationTime, size) - if err != nil { - return nil, err - } - // get the latest version of snapshot content before update status - latestContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(content.Name, metav1.GetOptions{}) - err = ctrl.updateSnapshotContentStatus(latestContent, snapshotID, readyToUse, creationTime.UnixNano(), size) - if err != nil { - return nil, err - } - // store latest content - _, err = ctrl.storeContentUpdate(latestContent) - if err != nil { - return nil, err - } - return newSnapshot, nil -} - -// The function goes through the whole snapshot creation process. -// 1. Trigger the snapshot through csi storage provider. -// 2. Update VolumeSnapshot status with creationtimestamp information -// 3. Create the VolumeSnapshotContent object with the snapshot id information. -// 4. Bind the VolumeSnapshot and VolumeSnapshotContent object -func (ctrl *csiSnapshotController) createSnapshotOperation(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshot, error) { - klog.Infof("createSnapshot: Creating snapshot %s through the plugin ...", snapshotKey(snapshot)) - - if snapshot.Status != nil && snapshot.Status.Error != nil && snapshot.Status.Error.Message != nil && !isControllerUpdateFailError(snapshot.Status.Error) { - klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *snapshot.Status.Error.Message) - return snapshot, nil - } - - // If PVC is not being deleted and finalizer is not added yet, a finalizer should be added. - klog.V(5).Infof("createSnapshotOperation: Check if PVC is not being deleted and add Finalizer for source of snapshot [%s] if needed", snapshot.Name) - err := ctrl.ensureSnapshotSourceFinalizer(snapshot) - if err != nil { - klog.Errorf("createSnapshotOperation failed to add finalizer for source of snapshot %s", err) - return nil, err - } - - class, volume, contentName, snapshotterSecretRef, err := ctrl.getCreateSnapshotInput(snapshot) - if err != nil { - return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", snapshot.Name, err) - } - - snapshotterCredentials, err := getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return nil, err - } - - driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(snapshot, volume, class.Parameters, snapshotterCredentials) - if err != nil { - return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", volume.Name, err) - } - - klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) - - var newSnapshot *crdv1.VolumeSnapshot - // Update snapshot status with creationTime - for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { - klog.V(5).Infof("createSnapshot [%s]: trying to update snapshot creation timestamp", snapshotKey(snapshot)) - newSnapshot, err = ctrl.updateSnapshotStatus(snapshot, contentName, readyToUse, creationTime, size) - if err == nil { - break - } - klog.V(4).Infof("failed to update snapshot %s creation timestamp: %v", snapshotKey(snapshot), err) - } - - if err != nil { - return nil, err - } - // Create VolumeSnapshotContent in the database - snapshotRef, err := ref.GetReference(scheme.Scheme, snapshot) - if err != nil { - return nil, err - } - - timestamp := creationTime.UnixNano() - snapshotContent := &crdv1.VolumeSnapshotContent{ - ObjectMeta: metav1.ObjectMeta{ - Name: contentName, - }, - Spec: crdv1.VolumeSnapshotContentSpec{ - VolumeSnapshotRef: *snapshotRef, - DeletionPolicy: class.DeletionPolicy, - Driver: driverName, - VolumeSnapshotClassName: &class.Name, - Source: crdv1.VolumeSnapshotContentSource{ - VolumeHandle: &volume.Spec.CSI.VolumeHandle, - }, - }, - } - - // Set AnnDeletionSecretRefName and AnnDeletionSecretRefNamespace - if snapshotterSecretRef != nil { - klog.V(5).Infof("createSnapshotOperation: set annotation [%s] on content [%s].", AnnDeletionSecretRefName, snapshotContent.Name) - metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, AnnDeletionSecretRefName, snapshotterSecretRef.Name) - - klog.V(5).Infof("syncContent: set annotation [%s] on content [%s].", AnnDeletionSecretRefNamespace, snapshotContent.Name) - metav1.SetMetaDataAnnotation(&snapshotContent.ObjectMeta, AnnDeletionSecretRefNamespace, snapshotterSecretRef.Namespace) - } - - klog.V(3).Infof("volume snapshot content %v", snapshotContent) - // Try to create the VolumeSnapshotContent object several times - for i := 0; i < ctrl.createSnapshotContentRetryCount; i++ { - klog.V(5).Infof("createSnapshot [%s]: trying to save volume snapshot content %s", snapshotKey(snapshot), snapshotContent.Name) - if _, err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Create(snapshotContent); err == nil || apierrs.IsAlreadyExists(err) { - // creation succeeded. - if err != nil { - klog.V(3).Infof("volume snapshot content %q for snapshot %q already exists, reusing", snapshotContent.Name, snapshotKey(snapshot)) - err = nil - } else { - klog.V(3).Infof("volume snapshot content %q for snapshot %q saved, %v", snapshotContent.Name, snapshotKey(snapshot), snapshotContent) - } - newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(snapshotContent.Name, metav1.GetOptions{}) - if err == nil { - err = ctrl.updateSnapshotContentStatus(newContent, snapshotID, readyToUse, timestamp, size) - } - break - } - // Save failed, try again after a while. - klog.V(3).Infof("failed to save volume snapshot content %q for snapshot %q: %v", snapshotContent.Name, snapshotKey(snapshot), err) - time.Sleep(ctrl.createSnapshotContentInterval) - } - - if err != nil { - // Save failed. Now we have a snapshot asset outside of Kubernetes, - // but we don't have appropriate volumesnapshot content object for it. - // Emit some event here and controller should try to create the content in next sync period. - strerr := fmt.Sprintf("Error creating volume snapshot content object for snapshot %s: %v.", snapshotKey(snapshot), err) - klog.Error(strerr) - ctrl.eventRecorder.Event(newSnapshot, v1.EventTypeWarning, "CreateSnapshotContentFailed", strerr) - return nil, newControllerUpdateError(snapshotKey(snapshot), err.Error()) - } - return newSnapshot, nil -} - -// Delete a snapshot -// 1. Find the SnapshotContent corresponding to Snapshot -// 1a: Not found => finish (it's been deleted already) -// 2. Ask the backend to remove the snapshot device -// 3. Delete the SnapshotContent object -// 4. Remove the Snapshot from store -// 5. Finish -func (ctrl *csiSnapshotController) deleteSnapshotContentOperation(content *crdv1.VolumeSnapshotContent) error { - klog.V(5).Infof("deleteSnapshotOperation [%s] started", content.Name) - - // get secrets if VolumeSnapshotClass specifies it - var snapshotterCredentials map[string]string - /* TODO(@Xing-yang): secrete ref retrivial needs to be implemented here - snapshotClassName := content.Spec.VolumeSnapshotClassName - if snapshotClassName != nil { - if snapshotClass, err := ctrl.classLister.Get(*snapshotClassName); err == nil { - // Resolve snapshotting secret credentials. - // No VolumeSnapshot is provided when resolving delete secret names, since the VolumeSnapshot may or may not exist at delete time. - snapshotterSecretRef, err := getSecretReference(snapshotClass.Parameters, content.Name, nil) - if err != nil { - return err - } - snapshotterCredentials, err = getCredentials(ctrl.client, snapshotterSecretRef) - if err != nil { - return err - } - } - }*/ - - err := ctrl.handler.DeleteSnapshot(content, snapshotterCredentials) - if err != nil { - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to delete snapshot") - return fmt.Errorf("failed to delete snapshot %#v, err: %v", content.Name, err) - } - - err = ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Delete(content.Name, &metav1.DeleteOptions{}) - if err != nil { - ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotContentObjectDeleteError", "Failed to delete snapshot content API object") - return fmt.Errorf("failed to delete VolumeSnapshotContent %s from API server: %q", content.Name, err) - } - - return nil -} - -// TODO(xiangqian) This is a temp implementation to make sure the test cases pass -// updateSnapshotContentStatus updates status of a content object -func (ctrl *csiSnapshotController) updateSnapshotContentStatus( - content *crdv1.VolumeSnapshotContent, - snapshotHandle string, - readyToUse bool, - createdAt int64, - size int64) error { - var newStatus *crdv1.VolumeSnapshotContentStatus - updated := false - if content.Status == nil { - newStatus = &crdv1.VolumeSnapshotContentStatus{ - SnapshotHandle: &snapshotHandle, - ReadyToUse: &readyToUse, - CreationTime: &createdAt, - RestoreSize: &size, - } - updated = true - } else { - newStatus = content.Status.DeepCopy() - if newStatus.SnapshotHandle == nil { - newStatus.SnapshotHandle = &snapshotHandle - updated = true - } - if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { - newStatus.ReadyToUse = &readyToUse - updated = true - if readyToUse && newStatus.Error != nil { - newStatus.Error = nil - } - } - if newStatus.CreationTime == nil { - newStatus.CreationTime = &createdAt - updated = true - } - if newStatus.RestoreSize == nil { - newStatus.RestoreSize = &size - updated = true - } - } - - if updated { - contentClone := content.DeepCopy() - contentClone.Status = newStatus - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - } - return nil -} - -// UpdateSnapshotStatus converts snapshot status to crdv1.VolumeSnapshotCondition -func (ctrl *csiSnapshotController) updateSnapshotStatus( - snapshot *crdv1.VolumeSnapshot, - boundContentName string, - readyToUse bool, - createdAt time.Time, - size int64) (*crdv1.VolumeSnapshot, error) { - klog.V(5).Infof("updating VolumeSnapshot[]%s, readyToUse %v, timestamp %v", snapshotKey(snapshot), readyToUse, createdAt) - - var newStatus *crdv1.VolumeSnapshotStatus - updated := false - if snapshot.Status == nil { - newStatus = &crdv1.VolumeSnapshotStatus{ - BoundVolumeSnapshotContentName: &boundContentName, - CreationTime: &metav1.Time{Time: createdAt}, - ReadyToUse: &readyToUse, - RestoreSize: resource.NewQuantity(size, resource.BinarySI), - } - updated = true - } else { - newStatus = snapshot.Status.DeepCopy() - if newStatus.BoundVolumeSnapshotContentName == nil { - newStatus.BoundVolumeSnapshotContentName = &boundContentName - updated = true - } - if newStatus.CreationTime == nil { - newStatus.CreationTime = &metav1.Time{Time: createdAt} - updated = true - } - if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { - newStatus.ReadyToUse = &readyToUse - updated = true - if readyToUse && newStatus.Error != nil { - newStatus.Error = nil - } - } - if newStatus.RestoreSize == nil { - newStatus.RestoreSize = resource.NewQuantity(size, resource.BinarySI) - updated = true - } - } - - if updated { - snapshotClone := snapshot.DeepCopy() - snapshotClone.Status = newStatus - newSnapshotObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).UpdateStatus(snapshotClone) - if err != nil { - return nil, newControllerUpdateError(snapshotKey(snapshot), err.Error()) - } - return newSnapshotObj, nil - } - return snapshot, nil -} - -// getVolumeFromVolumeSnapshot is a helper function to get PV from VolumeSnapshot. -func (ctrl *csiSnapshotController) getVolumeFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolume, error) { - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - - if pvc.Status.Phase != v1.ClaimBound { - return nil, fmt.Errorf("the PVC %s is not yet bound to a PV, will not attempt to take a snapshot", pvc.Name) - } - - pvName := pvc.Spec.VolumeName - pv, err := ctrl.client.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("failed to retrieve PV %s from the API server: %q", pvName, err) - } - - klog.V(5).Infof("getVolumeFromVolumeSnapshot: snapshot [%s] PV name [%s]", snapshot.Name, pvName) - - return pv, nil -} - -func (ctrl *csiSnapshotController) getStorageClassFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*storagev1.StorageClass, error) { - // Get storage class from PVC or PV - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - storageclassName := *pvc.Spec.StorageClassName - if len(storageclassName) == 0 { - volume, err := ctrl.getVolumeFromVolumeSnapshot(snapshot) - if err != nil { - return nil, err - } - storageclassName = volume.Spec.StorageClassName - } - if len(storageclassName) == 0 { - return nil, fmt.Errorf("cannot figure out the snapshot class automatically, please specify one in snapshot spec") - } - storageclass, err := ctrl.client.StorageV1().StorageClasses().Get(storageclassName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - return storageclass, nil -} - -// GetSnapshotClass is a helper function to get snapshot class from the class name. -func (ctrl *csiSnapshotController) GetSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { - klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) - - class, err := ctrl.classLister.Get(className) - if err != nil { - klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) - return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) - } - - return class, nil -} - -// SetDefaultSnapshotClass is a helper function to figure out the default snapshot class from -// PVC/PV StorageClass and update VolumeSnapshot with this snapshot class name. -func (ctrl *csiSnapshotController) SetDefaultSnapshotClass(snapshot *crdv1.VolumeSnapshot) (*crdv1.VolumeSnapshotClass, *crdv1.VolumeSnapshot, error) { - klog.V(5).Infof("SetDefaultSnapshotClass for snapshot [%s]", snapshot.Name) - - storageclass, err := ctrl.getStorageClassFromVolumeSnapshot(snapshot) - if err != nil { - return nil, nil, err - } - // Find default snapshot class if available - list, err := ctrl.classLister.List(labels.Everything()) - if err != nil { - return nil, nil, err - } - defaultClasses := []*crdv1.VolumeSnapshotClass{} - - for _, class := range list { - if IsDefaultAnnotation(class.ObjectMeta) && storageclass.Provisioner == class.Driver && ctrl.driverName == class.Driver { - defaultClasses = append(defaultClasses, class) - klog.V(5).Infof("get defaultClass added: %s", class.Name) - } - } - if len(defaultClasses) == 0 { - return nil, nil, fmt.Errorf("cannot find default snapshot class") - } - if len(defaultClasses) > 1 { - klog.V(4).Infof("get DefaultClass %d defaults found", len(defaultClasses)) - return nil, nil, fmt.Errorf("%d default snapshot classes were found", len(defaultClasses)) - } - klog.V(5).Infof("setDefaultSnapshotClass [%s]: default VolumeSnapshotClassName [%s]", snapshot.Name, defaultClasses[0].Name) - snapshotClone := snapshot.DeepCopy() - snapshotClone.Spec.VolumeSnapshotClassName = &(defaultClasses[0].Name) - newSnapshot, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - klog.V(4).Infof("updating VolumeSnapshot[%s] default class failed %v", snapshotKey(snapshot), err) - } - _, updateErr := ctrl.storeSnapshotUpdate(newSnapshot) - if updateErr != nil { - // We will get an "snapshot update" event soon, this is not a big error - klog.V(4).Infof("setDefaultSnapshotClass [%s]: cannot update internal cache: %v", snapshotKey(snapshot), updateErr) - } - - return defaultClasses[0], newSnapshot, nil -} - -// getClaimFromVolumeSnapshot is a helper function to get PVC from VolumeSnapshot. -func (ctrl *csiSnapshotController) getClaimFromVolumeSnapshot(snapshot *crdv1.VolumeSnapshot) (*v1.PersistentVolumeClaim, error) { - if snapshot.Spec.Source.PersistentVolumeClaimName == nil { - return nil, fmt.Errorf("the snapshot source is not the right type. Expected PersistentVolumeClaimName to be valid") - } - pvcName := *snapshot.Spec.Source.PersistentVolumeClaimName - if pvcName == "" { - return nil, fmt.Errorf("the PVC name is not specified in snapshot %s", snapshotKey(snapshot)) - } - - pvc, err := ctrl.pvcLister.PersistentVolumeClaims(snapshot.Namespace).Get(pvcName) - if err != nil { - return nil, fmt.Errorf("failed to retrieve PVC %s from the lister: %q", pvcName, err) - } - - return pvc, nil -} - -var _ error = controllerUpdateError{} - -type controllerUpdateError struct { - message string -} - -func newControllerUpdateError(name, message string) error { - return controllerUpdateError{ - message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), - } -} - -func (e controllerUpdateError) Error() string { - return e.message -} - -func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { - if err != nil && err.Message != nil { - if strings.Contains(*err.Message, controllerUpdateFailMsg) { - return true - } - } - return false -} - -// addContentFinalizer adds a Finalizer for VolumeSnapshotContent. -func (ctrl *csiSnapshotController) addContentFinalizer(content *crdv1.VolumeSnapshotContent) error { - contentClone := content.DeepCopy() - contentClone.ObjectMeta.Finalizers = append(contentClone.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - - _, err = ctrl.storeContentUpdate(contentClone) - if err != nil { - klog.Errorf("failed to update content store %v", err) - } - - klog.V(5).Infof("Added protection finalizer to volume snapshot content %s", content.Name) - return nil -} - -// removeContentFinalizer removes a Finalizer for VolumeSnapshotContent. -func (ctrl *csiSnapshotController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error { - contentClone := content.DeepCopy() - contentClone.ObjectMeta.Finalizers = slice.RemoveString(contentClone.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) - if err != nil { - return newControllerUpdateError(content.Name, err.Error()) - } - - _, err = ctrl.storeContentUpdate(contentClone) - if err != nil { - klog.Errorf("failed to update content store %v", err) - } - - klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name) - return nil -} - -// addSnapshotFinalizer adds a Finalizer for VolumeSnapshot. -func (ctrl *csiSnapshotController) addSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) error { - snapshotClone := snapshot.DeepCopy() - snapshotClone.ObjectMeta.Finalizers = append(snapshotClone.ObjectMeta.Finalizers, VolumeSnapshotFinalizer) - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - return newControllerUpdateError(snapshot.Name, err.Error()) - } - - _, err = ctrl.storeSnapshotUpdate(snapshotClone) - if err != nil { - klog.Errorf("failed to update snapshot store %v", err) - } - - klog.V(5).Infof("Added protection finalizer to volume snapshot %s", snapshotKey(snapshot)) - return nil -} - -// removeContentFinalizer removes a Finalizer for VolumeSnapshot. -func (ctrl *csiSnapshotController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) error { - snapshotClone := snapshot.DeepCopy() - snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) - - _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshots(snapshotClone.Namespace).Update(snapshotClone) - if err != nil { - return newControllerUpdateError(snapshot.Name, err.Error()) - } - - _, err = ctrl.storeSnapshotUpdate(snapshotClone) - if err != nil { - klog.Errorf("failed to update snapshot store %v", err) - } - - klog.V(5).Infof("Removed protection finalizer from volume snapshot %s", snapshotKey(snapshot)) - return nil -} - -// ensureSnapshotSourceFinalizer checks if a Finalizer needs to be added for the snapshot source; -// if true, adds a Finalizer for VolumeSnapshot Source PVC -func (ctrl *csiSnapshotController) ensureSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) - return nil - } - - // If PVC is not being deleted and PVCFinalizer is not added yet, the PVCFinalizer should be added. - if pvc.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { - // Add the finalizer - pvcClone := pvc.DeepCopy() - pvcClone.ObjectMeta.Finalizers = append(pvcClone.ObjectMeta.Finalizers, PVCFinalizer) - _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) - if err != nil { - klog.Errorf("cannot add finalizer on claim [%s] for snapshot [%s]: [%v]", pvc.Name, snapshot.Name, err) - return newControllerUpdateError(pvcClone.Name, err.Error()) - } - klog.Infof("Added protection finalizer to persistent volume claim %s", pvc.Name) - } - - return nil -} - -// removeSnapshotSourceFinalizer removes a Finalizer for VolumeSnapshot Source PVC. -func (ctrl *csiSnapshotController) removeSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) - return nil - } - - pvcClone := pvc.DeepCopy() - pvcClone.ObjectMeta.Finalizers = slice.RemoveString(pvcClone.ObjectMeta.Finalizers, PVCFinalizer, nil) - - _, err = ctrl.client.CoreV1().PersistentVolumeClaims(pvcClone.Namespace).Update(pvcClone) - if err != nil { - return newControllerUpdateError(pvcClone.Name, err.Error()) - } - - klog.V(5).Infof("Removed protection finalizer from persistent volume claim %s", pvc.Name) - return nil -} - -// isSnapshotSourceBeingUsed checks if a PVC is being used as a source to create a snapshot -func (ctrl *csiSnapshotController) isSnapshotSourceBeingUsed(snapshot *crdv1.VolumeSnapshot) bool { - klog.V(5).Infof("isSnapshotSourceBeingUsed[%s]: started", snapshotKey(snapshot)) - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("isSnapshotSourceBeingUsed: cannot to get claim from snapshot: %v", err) - return false - } - - // Going through snapshots in the cache (snapshotLister). If a snapshot's PVC source - // is the same as the input snapshot's PVC source and snapshot's ReadyToUse status - // is false, the snapshot is still being created from the PVC and the PVC is in-use. - snapshots, err := ctrl.snapshotLister.VolumeSnapshots(snapshot.Namespace).List(labels.Everything()) - if err != nil { - return false - } - for _, snap := range snapshots { - // Skip static bound snapshot without a PVC source - if snap.Spec.Source.PersistentVolumeClaimName == nil { - klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) - continue - } - if pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && !isSnapshotReadyToUse(snap) { - klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) - return true - } - } - - klog.V(5).Infof("isSnapshotSourceBeingUsed: no snapshot is being created from PVC %s/%s", pvc.Namespace, pvc.Name) - return false -} - -// checkandRemoveSnapshotSourceFinalizer checks if the snapshot source finalizer should be removed -// and removed it if needed. -func (ctrl *csiSnapshotController) checkandRemoveSnapshotSourceFinalizer(snapshot *crdv1.VolumeSnapshot) error { - // Get snapshot source which is a PVC - pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) - if err != nil { - klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already. No need to remove finalizer on the claim.", snapshot.Name, err) - return nil - } - - klog.V(5).Infof("checkandRemoveSnapshotSourceFinalizer for snapshot [%s]: snapshot status [%#v]", snapshot.Name, snapshot.Status) - - // Check if there is a Finalizer on PVC to be removed - if slice.ContainsString(pvc.ObjectMeta.Finalizers, PVCFinalizer, nil) { - // There is a Finalizer on PVC. Check if PVC is used - // and remove finalizer if it's not used. - isUsed := ctrl.isSnapshotSourceBeingUsed(snapshot) - if !isUsed { - klog.Infof("checkandRemoveSnapshotSourceFinalizer[%s]: Remove Finalizer for PVC %s as it is not used by snapshots in creation", snapshot.Name, pvc.Name) - err = ctrl.removeSnapshotSourceFinalizer(snapshot) - if err != nil { - klog.Errorf("checkandRemoveSnapshotSourceFinalizer [%s]: removeSnapshotSourceFinalizer failed to remove finalizer %v", snapshot.Name, err) - return err - } - } - } - - return nil -} - -// isSnapshotReadyToUse checks if a snapshot object has ReadyToUse in Status set to true -func isSnapshotReadyToUse(snapshot *crdv1.VolumeSnapshot) bool { - if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil { - return false - } - return *snapshot.Status.ReadyToUse -} diff --git a/pkg/controller/csi_handler.go b/pkg/sidecar-controller/csi_handler.go similarity index 62% rename from pkg/controller/csi_handler.go rename to pkg/sidecar-controller/csi_handler.go index 840ab150e..f261c8c97 100644 --- a/pkg/controller/csi_handler.go +++ b/pkg/sidecar-controller/csi_handler.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package sidecar_controller import ( "context" @@ -24,13 +24,12 @@ import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" - - v1 "k8s.io/api/core/v1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" ) // Handler is responsible for handling VolumeSnapshot events from informer. type Handler interface { - CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) + CreateSnapshot(content *crdv1.VolumeSnapshotContent, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error GetSnapshotStatus(content *crdv1.VolumeSnapshotContent) (bool, time.Time, int64, error) } @@ -58,27 +57,45 @@ func NewCSIHandler( } } -func (handler *csiHandler) CreateSnapshot(snapshot *crdv1.VolumeSnapshot, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { +func (handler *csiHandler) CreateSnapshot(content *crdv1.VolumeSnapshotContent, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - snapshotName, err := makeSnapshotName(handler.snapshotNamePrefix, string(snapshot.UID), handler.snapshotNameUUIDLength) + if content.Spec.VolumeSnapshotRef.UID == "" { + return "", "", time.Time{}, 0, false, fmt.Errorf("cannot create snapshot. Snapshot content %s not bound to a snapshot", content.Name) + } + + if content.Spec.Source.VolumeHandle == nil { + return "", "", time.Time{}, 0, false, fmt.Errorf("cannot create snapshot. Volume handle not found in snapshot content %s", content.Name) + } + + snapshotName, err := makeSnapshotName(handler.snapshotNamePrefix, string(content.Spec.VolumeSnapshotRef.UID), handler.snapshotNameUUIDLength) if err != nil { return "", "", time.Time{}, 0, false, err } - newParameters, err := removePrefixedParameters(parameters) + newParameters, err := utils.RemovePrefixedParameters(parameters) if err != nil { return "", "", time.Time{}, 0, false, fmt.Errorf("failed to remove CSI Parameters of prefixed keys: %v", err) } - return handler.snapshotter.CreateSnapshot(ctx, snapshotName, volume, newParameters, snapshotterCredentials) + return handler.snapshotter.CreateSnapshot(ctx, snapshotName, *content.Spec.Source.VolumeHandle, newParameters, snapshotterCredentials) } func (handler *csiHandler) DeleteSnapshot(content *crdv1.VolumeSnapshotContent, snapshotterCredentials map[string]string) error { ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - err := handler.snapshotter.DeleteSnapshot(ctx, *content.Status.SnapshotHandle, snapshotterCredentials) + var snapshotHandle string + var err error + if content.Status != nil && content.Status.SnapshotHandle != nil { + snapshotHandle = *content.Status.SnapshotHandle + } else if content.Spec.Source.SnapshotHandle != nil { + snapshotHandle = *content.Spec.Source.SnapshotHandle + } else { + return fmt.Errorf("failed to delete snapshot content %s: snapshotHandle is missing", content.Name) + } + + err = handler.snapshotter.DeleteSnapshot(ctx, snapshotHandle, snapshotterCredentials) if err != nil { return fmt.Errorf("failed to delete snapshot content %s: %q", content.Name, err) } @@ -90,9 +107,19 @@ func (handler *csiHandler) GetSnapshotStatus(content *crdv1.VolumeSnapshotConten ctx, cancel := context.WithTimeout(context.Background(), handler.timeout) defer cancel() - csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, *content.Status.SnapshotHandle) + var snapshotHandle string + var err error + if content.Status != nil && content.Status.SnapshotHandle != nil { + snapshotHandle = *content.Status.SnapshotHandle + } else if content.Spec.Source.SnapshotHandle != nil { + snapshotHandle = *content.Spec.Source.SnapshotHandle + } else { + return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: snapshotHandle is missing", content.Name) + } + + csiSnapshotStatus, timestamp, size, err := handler.snapshotter.GetSnapshotStatus(ctx, snapshotHandle) if err != nil { - return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot content %s: %q", content.Name, err) + return false, time.Time{}, 0, fmt.Errorf("failed to list snapshot for content %s: %q", content.Name, err) } return csiSnapshotStatus, timestamp, size, nil diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go new file mode 100644 index 000000000..c3199698c --- /dev/null +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -0,0 +1,539 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "fmt" + "strings" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/utils" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/goroutinemap" + "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" + "k8s.io/kubernetes/pkg/util/slice" +) + +// Design: +// +// This is the sidecar controller that is responsible for creating and deleting a +// snapshot on the storage infrastructure through a csi volume driver. It watches +// the VolumeSnapshotContent object which is either created/deleted by the +// common snapshot controller in the case of dynamic provisioning or by the admin +// in the case of pre-provisioned snapshots. + +// The snapshot creation through csi volume driver should return a snapshot after +// it is created successfully (however, the snapshot might not be ready to use yet if +// there is an uploading phase). The creationTime will be updated accordingly +// on the status of VolumeSnapshotContent. +// After that, the sidecar controller will keep checking the snapshot status +// through csi snapshot calls. When the snapshot is ready to use, the sidecar +// controller set the status "ReadyToUse" to true on the VolumeSnapshotContent object +// to indicate the snapshot is ready to use. If the creation failed for any reason, +// the Error status is set accordingly. + +const controllerUpdateFailMsg = "snapshot controller failed to update" + +// syncContent deals with one key off the queue. It returns false when it's time to quit. +func (ctrl *csiSnapshotSideCarController) syncContent(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("synchronizing VolumeSnapshotContent[%s]", content.Name) + + var err error + if ctrl.shouldDelete(content) { + switch content.Spec.DeletionPolicy { + case crdv1.VolumeSnapshotContentRetain: + klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Retain. Keep physical snapshot and remove content finalizer", content.Name) + // It is a deletion candidate if DeletionTimestamp is not nil and + // VolumeSnapshotContentFinalizer is set. + if utils.IsContentDeletionCandidate(content) { + // Volume snapshot content is a deletion candidate. + // Remove the content finalizer. + klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name) + return ctrl.removeContentFinalizer(content) + } + + case crdv1.VolumeSnapshotContentDelete: + klog.V(4).Infof("VolumeSnapshotContent[%s]: policy is Delete. Delete physical snapshot", content.Name) + err = ctrl.deleteCSISnapshot(content) + if err != nil { + return err + } + klog.V(5).Infof("syncContent: check if we should remove Finalizer for VolumeSnapshotContent[%s]", content.Name) + // It is a deletion candidate if DeletionTimestamp is not nil and + // VolumeSnapshotContentFinalizer is set. + if utils.IsContentDeletionCandidate(content) { + // Volume snapshot content is a deletion candidate. + // Remove the content finalizer. + klog.V(5).Infof("syncContent: Content [%s] is a deletion candidate. Remove finalizer.", content.Name) + return ctrl.removeContentFinalizer(content) + } + + default: + // Unknown VolumeSnapshotDeletionPolicy + ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotUnknownDeletionPolicy", "Volume Snapshot Content has unrecognized deletion policy") + } + klog.V(4).Infof("VolumeSnapshotContent[%s]: the policy is %s", content.Name, content.Spec.DeletionPolicy) + } else { + var err error + klog.V(5).Infof("syncContent: Call CreateSnapshot for content %s", content.Name) + if content.Spec.Source.VolumeHandle != nil && content.Status == nil { + if err = ctrl.createSnapshot(content); err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot with error %v", err)) + return err + } + } else { + + if err = ctrl.checkandUpdateContentStatus(content); err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentStatusUpdateFailed", fmt.Sprintf("Failed to update snapshot content status with error %v", err)) + return err + } + } + + return nil + } + + return nil +} + +// deleteCSISnapshot starts delete action. +func (ctrl *csiSnapshotSideCarController) deleteCSISnapshot(content *crdv1.VolumeSnapshotContent) error { + operationName := fmt.Sprintf("delete-%s", content.Name) + klog.V(5).Infof("Snapshotter is about to delete volume snapshot content and the operation named %s", operationName) + ctrl.scheduleOperation(operationName, func() error { + return ctrl.deleteCSISnapshotOperation(content) + }) + return nil +} + +// scheduleOperation starts given asynchronous operation on given volume. It +// makes sure the operation is already not running. +func (ctrl *csiSnapshotSideCarController) scheduleOperation(operationName string, operation func() error) { + klog.V(5).Infof("scheduleOperation[%s]", operationName) + + err := ctrl.runningOperations.Run(operationName, operation) + if err != nil { + switch { + case goroutinemap.IsAlreadyExists(err): + klog.V(4).Infof("operation %q is already running, skipping", operationName) + case exponentialbackoff.IsExponentialBackoff(err): + klog.V(4).Infof("operation %q postponed due to exponential backoff", operationName) + default: + klog.Errorf("error scheduling operation %q: %v", operationName, err) + } + } +} + +func (ctrl *csiSnapshotSideCarController) storeContentUpdate(content interface{}) (bool, error) { + return utils.StoreObjectUpdate(ctrl.contentStore, content, "content") +} + +// createSnapshot starts new asynchronous operation to create snapshot +func (ctrl *csiSnapshotSideCarController) createSnapshot(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("createSnapshot for content [%s]: started", content.Name) + opName := fmt.Sprintf("create-%s", content.Name) + ctrl.scheduleOperation(opName, func() error { + contentObj, err := ctrl.createSnapshotOperation(content) + if err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotCreationFailed", fmt.Sprintf("Failed to create snapshot: %v", err)) + klog.Errorf("createSnapshot [%s]: error occurred in createSnapshotOperation: %v", opName, err) + return err + } + + _, updateErr := ctrl.storeContentUpdate(contentObj) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("createSnapshot [%s]: cannot update internal content cache: %v", content.Name, updateErr) + } + return nil + }) + return nil +} + +func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatus(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("checkandUpdateContentStatus[%s] started", content.Name) + opName := fmt.Sprintf("check-%s", content.Name) + ctrl.scheduleOperation(opName, func() error { + contentObj, err := ctrl.checkandUpdateContentStatusOperation(content) + if err != nil { + ctrl.updateContentErrorStatusWithEvent(content, v1.EventTypeWarning, "SnapshotContentCheckandUpdateFailed", fmt.Sprintf("Failed to check and update snapshot content: %v", err)) + klog.Errorf("checkandUpdateContentStatus [%s]: error occured %v", content.Name, err) + return err + } + _, updateErr := ctrl.storeContentUpdate(contentObj) + if updateErr != nil { + // We will get an "snapshot update" event soon, this is not a big error + klog.V(4).Infof("checkandUpdateContentStatus [%s]: cannot update internal cache: %v", content.Name, updateErr) + } + + return nil + }) + return nil +} + +// updateContentStatusWithEvent saves new content.Status to API server and emits +// given event on the content. It saves the status and emits the event only when +// the status has actually changed from the version saved in API server. +// Parameters: +// content - content to update +// eventtype, reason, message - event to send, see EventRecorder.Event() +func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(content *crdv1.VolumeSnapshotContent, eventtype, reason, message string) error { + klog.V(5).Infof("updateContentStatusWithEvent[%s]", content.Name) + + if content.Status != nil && content.Status.Error != nil && *content.Status.Error.Message == message { + klog.V(4).Infof("updateContentStatusWithEvent[%s]: the same error %v is already set", content.Name, content.Status.Error) + return nil + } else if content.Status == nil { + content.Status = &crdv1.VolumeSnapshotContentStatus{} + } + contentClone := content.DeepCopy() + statusError := &crdv1.VolumeSnapshotError{ + Time: &metav1.Time{ + Time: time.Now(), + }, + Message: &message, + } + contentClone.Status.Error = statusError + ready := false + contentClone.Status.ReadyToUse = &ready + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) + + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status failed %v", content.Name, err) + return err + } + + _, err = ctrl.storeContentUpdate(newContent) + if err != nil { + klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) + return err + } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newContent, eventtype, reason, message) + + return nil +} + +func (ctrl *csiSnapshotSideCarController) getCSISnapshotInput(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotClass, map[string]string, error) { + className := content.Spec.VolumeSnapshotClassName + klog.V(5).Infof("getCSISnapshotInput for content [%s]: VolumeSnapshotClassName [%s]", content.Name, *className) + var class *crdv1.VolumeSnapshotClass + var err error + if className != nil { + class, err = ctrl.getSnapshotClass(*className) + if err != nil { + klog.Errorf("getCSISnapshotInput failed to getClassFromVolumeSnapshot %s", err) + return nil, nil, err + } + } else { + // If dynamic provisioning, return failure if no snapshot class + if content.Spec.Source.VolumeHandle != nil { + klog.Errorf("failed to getCSISnapshotInput %s without a snapshot class", content.Name) + return nil, nil, fmt.Errorf("failed to take snapshot %s without a snapshot class", content.Name) + } + // For pre-provisioned snapshot, snapshot class is not required + klog.V(5).Infof("getCSISnapshotInput for content [%s]: no VolumeSnapshotClassName provided for pre-provisioned snapshot", content.Name) + return nil, nil, nil + } + + // Resolve snapshotting secret credentials. + snapshotterCredentials, err := ctrl.GetCredentialsFromAnnotation(content) + if err != nil { + return nil, nil, err + } + + return class, snapshotterCredentials, nil +} + +func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + var err error + var creationTime time.Time + var size int64 + var readyToUse = false + var driverName string + var snapshotID string + + if content.Spec.Source.SnapshotHandle != nil { + klog.V(5).Infof("checkandUpdateContentStatusOperation: call GetSnapshotStatus for snapshot which is pre-bound to content [%s]", content.Name) + readyToUse, creationTime, size, err = ctrl.handler.GetSnapshotStatus(content) + if err != nil { + klog.Errorf("checkandUpdateContentStatusOperation: failed to call get snapshot status to check whether snapshot is ready to use %q", err) + return nil, err + } + driverName = content.Spec.Driver + if content.Spec.Source.SnapshotHandle != nil { + snapshotID = *content.Spec.Source.SnapshotHandle + } + } else { + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot %s: %q", content.Name, err) + } + + driverName, snapshotID, creationTime, size, readyToUse, err = ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) + if err != nil { + klog.Errorf("checkandUpdateContentStatusOperation: failed to call create snapshot to check whether the snapshot is ready to use %q", err) + return nil, err + } + } + klog.V(5).Infof("checkandUpdateContentStatusOperation: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) + + if creationTime.IsZero() { + creationTime = time.Now() + } + + updateContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, creationTime.UnixNano(), size) + if err != nil { + return nil, err + } + return updateContent, nil +} + +// The function goes through the whole snapshot creation process. +// 1. Trigger the snapshot through csi storage provider. +// 2. Update VolumeSnapshot status with creationtimestamp information +// 3. Create the VolumeSnapshotContent object with the snapshot id information. +// 4. Bind the VolumeSnapshot and VolumeSnapshotContent object +func (ctrl *csiSnapshotSideCarController) createSnapshotOperation(content *crdv1.VolumeSnapshotContent) (*crdv1.VolumeSnapshotContent, error) { + klog.Infof("createSnapshotOperation: Creating snapshot for content %s through the plugin ...", content.Name) + + // content.Status will be created for the first time after a snapshot + // is created by the CSI driver. If content.Status is not nil, + // we should update content status without creating snapshot again. + if content.Status != nil && content.Status.Error != nil && content.Status.Error.Message != nil && !isControllerUpdateFailError(content.Status.Error) { + klog.V(4).Infof("error is already set in snapshot, do not retry to create: %s", *content.Status.Error.Message) + return content, nil + } + + class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return nil, fmt.Errorf("failed to get input parameters to create snapshot for content %s: %q", content.Name, err) + } + + driverName, snapshotID, creationTime, size, readyToUse, err := ctrl.handler.CreateSnapshot(content, class.Parameters, snapshotterCredentials) + if err != nil { + return nil, fmt.Errorf("failed to take snapshot of the volume, %s: %q", *content.Spec.Source.VolumeHandle, err) + } + if driverName != class.Driver { + return nil, fmt.Errorf("failed to take snapshot of the volume, %s: driver name %s returned from the driver is different from driver %s in snapshot class", *content.Spec.Source.VolumeHandle, driverName, class.Driver) + } + + klog.V(5).Infof("Created snapshot: driver %s, snapshotId %s, creationTime %v, size %d, readyToUse %t", driverName, snapshotID, creationTime, size, readyToUse) + + timestamp := creationTime.UnixNano() + newContent, err := ctrl.updateSnapshotContentStatus(content, snapshotID, readyToUse, timestamp, size) + if err != nil { + strerr := fmt.Sprintf("error updating volume snapshot content status for snapshot %s: %v.", content.Name, err) + klog.Error(strerr) + } else { + content = newContent + } + + // Update content in the cache store + _, err = ctrl.storeContentUpdate(content) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + return content, nil +} + +// Delete a snapshot: Ask the backend to remove the snapshot device +func (ctrl *csiSnapshotSideCarController) deleteCSISnapshotOperation(content *crdv1.VolumeSnapshotContent) error { + klog.V(5).Infof("deleteCSISnapshotOperation [%s] started", content.Name) + + _, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) + if err != nil { + return fmt.Errorf("failed to get input parameters to delete snapshot for content %s: %q", content.Name, err) + } + + err = ctrl.handler.DeleteSnapshot(content, snapshotterCredentials) + if err != nil { + ctrl.eventRecorder.Event(content, v1.EventTypeWarning, "SnapshotDeleteError", "Failed to delete snapshot") + return fmt.Errorf("failed to delete snapshot %#v, err: %v", content.Name, err) + } + + return nil +} + +func (ctrl *csiSnapshotSideCarController) updateSnapshotContentStatus( + content *crdv1.VolumeSnapshotContent, + snapshotHandle string, + readyToUse bool, + createdAt int64, + size int64) (*crdv1.VolumeSnapshotContent, error) { + klog.V(5).Infof("updateSnapshotContentStatus: updating VolumeSnapshotContent [%s], snapshotHandle %s, readyToUse %v, createdAt %v, size %d", content.Name, snapshotHandle, readyToUse, createdAt, size) + + contentObj, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Get(content.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("error get snapshot content %s from api server: %v", content.Name, err) + } + + var newStatus *crdv1.VolumeSnapshotContentStatus + updated := false + if contentObj.Status == nil { + newStatus = &crdv1.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + ReadyToUse: &readyToUse, + CreationTime: &createdAt, + RestoreSize: &size, + } + updated = true + } else { + newStatus = contentObj.Status.DeepCopy() + if newStatus.SnapshotHandle == nil { + newStatus.SnapshotHandle = &snapshotHandle + updated = true + } + if newStatus.ReadyToUse == nil || *newStatus.ReadyToUse != readyToUse { + newStatus.ReadyToUse = &readyToUse + updated = true + if readyToUse && newStatus.Error != nil { + newStatus.Error = nil + } + } + if newStatus.CreationTime == nil { + newStatus.CreationTime = &createdAt + updated = true + } + if newStatus.RestoreSize == nil { + newStatus.RestoreSize = &size + updated = true + } + } + + if updated { + contentClone := contentObj.DeepCopy() + contentClone.Status = newStatus + newContent, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().UpdateStatus(contentClone) + if err != nil { + return nil, newControllerUpdateError(content.Name, err.Error()) + } + return newContent, nil + } + + return contentObj, nil +} + +// getSnapshotClass is a helper function to get snapshot class from the class name. +func (ctrl *csiSnapshotSideCarController) getSnapshotClass(className string) (*crdv1.VolumeSnapshotClass, error) { + klog.V(5).Infof("getSnapshotClass: VolumeSnapshotClassName [%s]", className) + + class, err := ctrl.classLister.Get(className) + if err != nil { + klog.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + return nil, fmt.Errorf("failed to retrieve snapshot class %s from the informer: %q", className, err) + } + + return class, nil +} + +var _ error = controllerUpdateError{} + +type controllerUpdateError struct { + message string +} + +func newControllerUpdateError(name, message string) error { + return controllerUpdateError{ + message: fmt.Sprintf("%s %s on API server: %s", controllerUpdateFailMsg, name, message), + } +} + +func (e controllerUpdateError) Error() string { + return e.message +} + +func isControllerUpdateFailError(err *crdv1.VolumeSnapshotError) bool { + if err != nil { + if strings.Contains(*err.Message, controllerUpdateFailMsg) { + return true + } + } + return false +} + +func (ctrl *csiSnapshotSideCarController) GetCredentialsFromAnnotation(content *crdv1.VolumeSnapshotContent) (map[string]string, error) { + // get secrets if VolumeSnapshotClass specifies it + var snapshotterCredentials map[string]string + var err error + + // Check if annotation exists + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefName) && metav1.HasAnnotation(content.ObjectMeta, utils.AnnDeletionSecretRefNamespace) { + annDeletionSecretName := content.Annotations[utils.AnnDeletionSecretRefName] + annDeletionSecretNamespace := content.Annotations[utils.AnnDeletionSecretRefNamespace] + + snapshotterSecretRef := &v1.SecretReference{} + + if annDeletionSecretName == "" || annDeletionSecretNamespace == "" { + return nil, fmt.Errorf("cannot retrieve secrets for snapshot content %#v, err: secret name or namespace not specified", content.Name) + } + + snapshotterSecretRef.Name = annDeletionSecretName + snapshotterSecretRef.Namespace = annDeletionSecretNamespace + + snapshotterCredentials, err = utils.GetCredentials(ctrl.client, snapshotterSecretRef) + if err != nil { + // Continue with deletion, as the secret may have already been deleted. + klog.Errorf("Failed to get credentials for snapshot %s: %s", content.Name, err.Error()) + return nil, fmt.Errorf("cannot get credentials for snapshot content %#v", content.Name) + } + } + + return snapshotterCredentials, nil +} + +// removeContentFinalizer removes a Finalizer for VolumeSnapshotContent. +func (ctrl csiSnapshotSideCarController) removeContentFinalizer(content *crdv1.VolumeSnapshotContent) error { + contentClone := content.DeepCopy() + contentClone.ObjectMeta.Finalizers = slice.RemoveString(contentClone.ObjectMeta.Finalizers, utils.VolumeSnapshotContentFinalizer, nil) + + _, err := ctrl.clientset.SnapshotV1beta1().VolumeSnapshotContents().Update(contentClone) + if err != nil { + return newControllerUpdateError(content.Name, err.Error()) + } + + _, err = ctrl.storeContentUpdate(contentClone) + if err != nil { + klog.Errorf("failed to update content store %v", err) + } + + klog.V(5).Infof("Removed protection finalizer from volume snapshot content %s", content.Name) + return nil +} + +// shouldDelete checks if content object should be deleted +// if DeletionTimestamp is set on the content +func (ctrl *csiSnapshotSideCarController) shouldDelete(content *crdv1.VolumeSnapshotContent) bool { + klog.V(5).Infof("Check if VolumeSnapshotContent[%s] should be deleted.", content.Name) + + if content.ObjectMeta.DeletionTimestamp == nil { + return false + } + // 1) shouldDelete returns true if content is not bound + // (VolumeSnapshotRef.UID == "") for pre-provisioned snapshot + if content.Spec.Source.SnapshotHandle != nil && content.Spec.VolumeSnapshotRef.UID == "" { + return true + } + // 2) shouldDelete returns true if AnnVolumeSnapshotBeingDeleted annotation is set + if metav1.HasAnnotation(content.ObjectMeta, utils.AnnVolumeSnapshotBeingDeleted) { + return true + } + return false +} diff --git a/pkg/sidecar-controller/snapshot_controller_base.go b/pkg/sidecar-controller/snapshot_controller_base.go new file mode 100644 index 000000000..7612b23cb --- /dev/null +++ b/pkg/sidecar-controller/snapshot_controller_base.go @@ -0,0 +1,280 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sidecar_controller + +import ( + "fmt" + "time" + + crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" + clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" + storageinformers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions/volumesnapshot/v1beta1" + storagelisters "github.com/kubernetes-csi/external-snapshotter/pkg/client/listers/volumesnapshot/v1beta1" + "github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + "k8s.io/kubernetes/pkg/util/goroutinemap" +) + +type csiSnapshotSideCarController struct { + clientset clientset.Interface + client kubernetes.Interface + driverName string + eventRecorder record.EventRecorder + contentQueue workqueue.RateLimitingInterface + + contentLister storagelisters.VolumeSnapshotContentLister + contentListerSynced cache.InformerSynced + classLister storagelisters.VolumeSnapshotClassLister + classListerSynced cache.InformerSynced + + contentStore cache.Store + + handler Handler + // Map of scheduled/running operations. + runningOperations goroutinemap.GoRoutineMap + + resyncPeriod time.Duration +} + +// NewCSISnapshotSideCarController returns a new *csiSnapshotSideCarController +func NewCSISnapshotSideCarController( + clientset clientset.Interface, + client kubernetes.Interface, + driverName string, + volumeSnapshotContentInformer storageinformers.VolumeSnapshotContentInformer, + volumeSnapshotClassInformer storageinformers.VolumeSnapshotClassInformer, + snapshotter snapshotter.Snapshotter, + timeout time.Duration, + resyncPeriod time.Duration, + snapshotNamePrefix string, + snapshotNameUUIDLength int, +) *csiSnapshotSideCarController { + broadcaster := record.NewBroadcaster() + broadcaster.StartLogging(klog.Infof) + broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) + var eventRecorder record.EventRecorder + eventRecorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("csi-snapshotter %s", driverName)}) + + ctrl := &csiSnapshotSideCarController{ + clientset: clientset, + client: client, + driverName: driverName, + eventRecorder: eventRecorder, + handler: NewCSIHandler(snapshotter, timeout, snapshotNamePrefix, snapshotNameUUIDLength), + runningOperations: goroutinemap.NewGoRoutineMap(true), + resyncPeriod: resyncPeriod, + contentStore: cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc), + contentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csi-snapshotter-content"), + } + + volumeSnapshotContentInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { ctrl.enqueueContentWork(obj) }, + UpdateFunc: func(oldObj, newObj interface{}) { ctrl.enqueueContentWork(newObj) }, + DeleteFunc: func(obj interface{}) { ctrl.enqueueContentWork(obj) }, + }, + ctrl.resyncPeriod, + ) + ctrl.contentLister = volumeSnapshotContentInformer.Lister() + ctrl.contentListerSynced = volumeSnapshotContentInformer.Informer().HasSynced + + ctrl.classLister = volumeSnapshotClassInformer.Lister() + ctrl.classListerSynced = volumeSnapshotClassInformer.Informer().HasSynced + + return ctrl +} + +func (ctrl *csiSnapshotSideCarController) Run(workers int, stopCh <-chan struct{}) { + defer ctrl.contentQueue.ShutDown() + + klog.Infof("Starting CSI snapshotter") + defer klog.Infof("Shutting CSI snapshotter") + + if !cache.WaitForCacheSync(stopCh, ctrl.contentListerSynced, ctrl.classListerSynced) { + klog.Errorf("Cannot sync caches") + return + } + + ctrl.initializeCaches(ctrl.contentLister) + + for i := 0; i < workers; i++ { + go wait.Until(ctrl.contentWorker, 0, stopCh) + } + + <-stopCh +} + +// enqueueContentWork adds snapshot content to given work queue. +func (ctrl *csiSnapshotSideCarController) enqueueContentWork(obj interface{}) { + // Beware of "xxx deleted" events + if unknown, ok := obj.(cache.DeletedFinalStateUnknown); ok && unknown.Obj != nil { + obj = unknown.Obj + } + if content, ok := obj.(*crdv1.VolumeSnapshotContent); ok { + objName, err := cache.DeletionHandlingMetaNamespaceKeyFunc(content) + if err != nil { + klog.Errorf("failed to get key from object: %v, %v", err, content) + return + } + klog.V(5).Infof("enqueued %q for sync", objName) + ctrl.contentQueue.Add(objName) + } +} + +// contentWorker processes items from contentQueue. It must run only once, +// syncContent is not assured to be reentrant. +func (ctrl *csiSnapshotSideCarController) contentWorker() { + workFunc := func() bool { + keyObj, quit := ctrl.contentQueue.Get() + if quit { + return true + } + defer ctrl.contentQueue.Done(keyObj) + key := keyObj.(string) + klog.V(5).Infof("contentWorker[%s]", key) + + _, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + klog.V(4).Infof("error getting name of snapshotContent %q to get snapshotContent from informer: %v", key, err) + return false + } + content, err := ctrl.contentLister.Get(name) + // The content still exists in informer cache, the event must have + // been add/update/sync + if err == nil { + if ctrl.isDriverMatch(content) { + ctrl.updateContentInCacheStore(content) + } + return false + } + if !errors.IsNotFound(err) { + klog.V(2).Infof("error getting content %q from informer: %v", key, err) + return false + } + + // The content is not in informer cache, the event must have been + // "delete" + contentObj, found, err := ctrl.contentStore.GetByKey(key) + if err != nil { + klog.V(2).Infof("error getting content %q from cache: %v", key, err) + return false + } + if !found { + // The controller has already processed the delete event and + // deleted the content from its cache + klog.V(2).Infof("deletion of content %q was already processed", key) + return false + } + content, ok := contentObj.(*crdv1.VolumeSnapshotContent) + if !ok { + klog.Errorf("expected content, got %+v", content) + return false + } + ctrl.deleteContentInCacheStore(content) + return false + } + + for { + if quit := workFunc(); quit { + klog.Infof("content worker queue shutting down") + return + } + } +} + +// verify whether the driver specified in VolumeSnapshotContent matches the controller's driver name +func (ctrl *csiSnapshotSideCarController) isDriverMatch(content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.Source.VolumeHandle == nil && content.Spec.Source.SnapshotHandle == nil { + // Skip this snapshot content if it does not have a valid source + return false + } + if content.Spec.Driver != ctrl.driverName { + // Skip this snapshot content if the driver does not match + return false + } + snapshotClassName := content.Spec.VolumeSnapshotClassName + if snapshotClassName != nil { + if snapshotClass, err := ctrl.classLister.Get(*snapshotClassName); err == nil { + if snapshotClass.Driver != ctrl.driverName { + return false + } + } + } + return true +} + +// updateContent runs in worker thread and handles "content added", +// "content updated" and "periodic sync" events. +func (ctrl *csiSnapshotSideCarController) updateContentInCacheStore(content *crdv1.VolumeSnapshotContent) { + // Store the new content version in the cache and do not process it if this is + // an old version. + new, err := ctrl.storeContentUpdate(content) + if err != nil { + klog.Errorf("%v", err) + } + if !new { + return + } + err = ctrl.syncContent(content) + if err != nil { + if errors.IsConflict(err) { + // Version conflict error happens quite often and the controller + // recovers from it easily. + klog.V(3).Infof("could not sync content %q: %+v", content.Name, err) + } else { + klog.Errorf("could not sync content %q: %+v", content.Name, err) + } + } +} + +// deleteContent runs in worker thread and handles "content deleted" event. +func (ctrl *csiSnapshotSideCarController) deleteContentInCacheStore(content *crdv1.VolumeSnapshotContent) { + _ = ctrl.contentStore.Delete(content) + klog.V(4).Infof("content %q deleted", content.Name) +} + +// initializeCaches fills all controller caches with initial data from etcd in +// order to have the caches already filled when first addSnapshot/addContent to +// perform initial synchronization of the controller. +func (ctrl *csiSnapshotSideCarController) initializeCaches(contentLister storagelisters.VolumeSnapshotContentLister) { + contentList, err := contentLister.List(labels.Everything()) + if err != nil { + klog.Errorf("CSISnapshotController can't initialize caches: %v", err) + return + } + for _, content := range contentList { + if ctrl.isDriverMatch(content) { + contentClone := content.DeepCopy() + if _, err = ctrl.storeContentUpdate(contentClone); err != nil { + klog.Errorf("error updating volume snapshot content cache: %v", err) + } + } + } + + klog.V(4).Infof("controller initialized") +} diff --git a/pkg/snapshotter/snapshotter.go b/pkg/snapshotter/snapshotter.go index 157f08972..068142126 100644 --- a/pkg/snapshotter/snapshotter.go +++ b/pkg/snapshotter/snapshotter.go @@ -27,14 +27,13 @@ import ( "google.golang.org/grpc" - "k8s.io/api/core/v1" "k8s.io/klog" ) // Snapshotter implements CreateSnapshot/DeleteSnapshot operations against a remote CSI driver. type Snapshotter interface { // CreateSnapshot creates a snapshot for a volume - CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, snapshotId string, timestamp time.Time, size int64, readyToUse bool, err error) + CreateSnapshot(ctx context.Context, snapshotName string, volumeHandle string, parameters map[string]string, snapshotterCredentials map[string]string) (driverName string, snapshotId string, timestamp time.Time, size int64, readyToUse bool, err error) // DeleteSnapshot deletes a snapshot from a volume DeleteSnapshot(ctx context.Context, snapshotID string, snapshotterCredentials map[string]string) (err error) @@ -53,12 +52,8 @@ func NewSnapshotter(conn *grpc.ClientConn) Snapshotter { } } -func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volume *v1.PersistentVolume, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { +func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volumeHandle string, parameters map[string]string, snapshotterCredentials map[string]string) (string, string, time.Time, int64, bool, error) { klog.V(5).Infof("CSI CreateSnapshot: %s", snapshotName) - if volume.Spec.CSI == nil { - return "", "", time.Time{}, 0, false, fmt.Errorf("CSIPersistentVolumeSource not defined in spec") - } - client := csi.NewControllerClient(s.conn) driverName, err := csirpc.GetDriverName(ctx, s.conn) @@ -67,7 +62,7 @@ func (s *snapshot) CreateSnapshot(ctx context.Context, snapshotName string, volu } req := csi.CreateSnapshotRequest{ - SourceVolumeId: volume.Spec.CSI.VolumeHandle, + SourceVolumeId: volumeHandle, Name: snapshotName, Parameters: parameters, Secrets: snapshotterCredentials, @@ -118,6 +113,8 @@ func (s *snapshot) isListSnapshotsSupported(ctx context.Context) (bool, error) { } func (s *snapshot) GetSnapshotStatus(ctx context.Context, snapshotID string) (bool, time.Time, int64, error) { + klog.V(5).Infof("GetSnapshotStatus: %s", snapshotID) + client := csi.NewControllerClient(s.conn) // If the driver does not support ListSnapshots, assume the snapshot ID is valid. diff --git a/pkg/snapshotter/snapshotter_test.go b/pkg/snapshotter/snapshotter_test.go index b0ff50f10..073394b45 100644 --- a/pkg/snapshotter/snapshotter_test.go +++ b/pkg/snapshotter/snapshotter_test.go @@ -79,7 +79,6 @@ func TestCreateSnapshot(t *testing.T) { } csiVolume := FakeCSIVolume() - volumeWithoutCSI := FakeVolume() defaultRequest := &csi.CreateSnapshotRequest{ Name: defaultName, @@ -135,7 +134,7 @@ func TestCreateSnapshot(t *testing.T) { tests := []struct { name string snapshotName string - volume *v1.PersistentVolume + volumeHandle string parameters map[string]string secrets map[string]string input *csi.CreateSnapshotRequest @@ -147,7 +146,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "success", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: defaultResponse, expectError: false, @@ -156,7 +155,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "attributes", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, parameters: defaultParameter, input: attributesRequest, output: defaultResponse, @@ -166,25 +165,17 @@ func TestCreateSnapshot(t *testing.T) { { name: "secrets", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, secrets: createSecrets, input: secretsRequest, output: defaultResponse, expectError: false, expectResult: result, }, - { - name: "fail for volume without csi source", - snapshotName: defaultName, - volume: volumeWithoutCSI, - input: nil, - output: nil, - expectError: true, - }, { name: "gRPC transient error", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: nil, injectError: codes.DeadlineExceeded, @@ -193,7 +184,7 @@ func TestCreateSnapshot(t *testing.T) { { name: "gRPC final error", snapshotName: defaultName, - volume: csiVolume, + volumeHandle: csiVolume.Spec.CSI.VolumeHandle, input: defaultRequest, output: nil, injectError: codes.NotFound, @@ -224,7 +215,7 @@ func TestCreateSnapshot(t *testing.T) { } s := NewSnapshotter(csiConn) - driverName, snapshotId, timestamp, size, readyToUse, err := s.CreateSnapshot(context.Background(), test.snapshotName, test.volume, test.parameters, test.secrets) + driverName, snapshotId, timestamp, size, readyToUse, err := s.CreateSnapshot(context.Background(), test.snapshotName, test.volumeHandle, test.parameters, test.secrets) if test.expectError && err == nil { t.Errorf("test %q: Expected error, got none", test.name) } @@ -509,29 +500,3 @@ func FakeCSIVolume() *v1.PersistentVolume { return &volume } - -func FakeVolume() *v1.PersistentVolume { - volume := v1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "fake-csi-volume", - }, - Spec: v1.PersistentVolumeSpec{ - ClaimRef: &v1.ObjectReference{ - Kind: "PersistentVolumeClaim", - APIVersion: "v1", - UID: types.UID("uid123"), - Namespace: "default", - Name: "test-claim", - }, - PersistentVolumeSource: v1.PersistentVolumeSource{ - GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}, - }, - StorageClassName: "default", - }, - Status: v1.PersistentVolumeStatus{ - Phase: v1.VolumeBound, - }, - } - - return &volume -} diff --git a/pkg/controller/util.go b/pkg/utils/util.go similarity index 73% rename from pkg/controller/util.go rename to pkg/utils/util.go index 963dd128c..50eb5c2d9 100644 --- a/pkg/controller/util.go +++ b/pkg/utils/util.go @@ -14,18 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package utils import ( "fmt" "strings" - "os" - "strconv" - "time" - crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -34,6 +30,9 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog" "k8s.io/kubernetes/pkg/util/slice" + "os" + "strconv" + "time" ) var ( @@ -60,12 +59,27 @@ const ( // [Deprecated] CSI Parameters that are put into fields but // NOT stripped from the parameters passed to CreateSnapshot - snapshotterSecretNameKey = "csiSnapshotterSecretName" - snapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" + SnapshotterSecretNameKey = "csiSnapshotterSecretName" + SnapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots - VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-protection" - VolumeSnapshotFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-protection" + VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" + // Name of finalizer on VolumeSnapshot that is being used as a source to create a PVC + VolumeSnapshotBoundFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-bound-protection" + // Name of finalizer on VolumeSnapshot that is used as a source to create a PVC + VolumeSnapshotAsSourceFinalizer = "snapshot.storage.kubernetes.io/volumesnapshot-as-source-protection" + // Name of finalizer on PVCs that is being used as a source to create VolumeSnapshots + PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-as-source-protection" + + IsDefaultSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" + + // AnnVolumeSnapshotBeingDeleted annotation applies to VolumeSnapshotContents. + // It indicates that the common snapshot controller has verified that volume + // snapshot has a deletion timestamp and is being deleted. + // Sidecar controller needs to check the deletion policy on the + // VolumeSnapshotContentand and decide whether to delete the volume snapshot + // backing the snapshot content. + AnnVolumeSnapshotBeingDeleted = "snapshot.storage.kubernetes.io/volumesnapshot-being-deleted" // Annotation for secret name and namespace will be added to the content // and used at snapshot content deletion time. @@ -75,20 +89,17 @@ const ( var snapshotterSecretParams = deprecatedSecretParamsMap{ name: "Snapshotter", - deprecatedSecretNameKey: snapshotterSecretNameKey, - deprecatedSecretNamespaceKey: snapshotterSecretNamespaceKey, + deprecatedSecretNameKey: SnapshotterSecretNameKey, + deprecatedSecretNamespaceKey: SnapshotterSecretNamespaceKey, secretNameKey: prefixedSnapshotterSecretNameKey, secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, } -// Name of finalizer on PVCs that have been used as a source to create VolumeSnapshots -const PVCFinalizer = "snapshot.storage.kubernetes.io/pvc-protection" - -func snapshotKey(vs *crdv1.VolumeSnapshot) string { +func SnapshotKey(vs *crdv1.VolumeSnapshot) string { return fmt.Sprintf("%s/%s", vs.Namespace, vs.Name) } -func snapshotRefKey(vsref v1.ObjectReference) string { +func SnapshotRefKey(vsref *v1.ObjectReference) string { return fmt.Sprintf("%s/%s", vsref.Namespace, vsref.Name) } @@ -96,7 +107,7 @@ func snapshotRefKey(vsref v1.ObjectReference) string { // callback (i.e. with events from etcd) or with an object modified by the // controller itself. Returns "true", if the cache was updated, false if the // object is an old version and should be ignored. -func storeObjectUpdate(store cache.Store, obj interface{}, className string) (bool, error) { +func StoreObjectUpdate(store cache.Store, obj interface{}, className string) (bool, error) { objName, err := keyFunc(obj) if err != nil { return false, fmt.Errorf("Couldn't get key for object %+v: %v", obj, err) @@ -232,7 +243,7 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap // - the nameTemplate or namespaceTemplate contains a token that cannot be resolved // - the resolved name is not a valid secret name // - the resolved namespace is not a valid namespace name -func getSecretReference(snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { +func GetSecretReference(snapshotClassParams map[string]string, snapContentName string, snapshot *crdv1.VolumeSnapshot) (*v1.SecretReference, error) { nameTemplate, namespaceTemplate, err := verifyAndGetSecretNameAndNamespaceTemplate(snapshotterSecretParams, snapshotClassParams) if err != nil { return nil, fmt.Errorf("failed to get name and namespace template from params: %v", err) @@ -244,7 +255,7 @@ func getSecretReference(snapshotClassParams map[string]string, snapContentName s ref := &v1.SecretReference{} - // Secret namespace template can make use of the VolumeSnapshotContent name or the VolumeSnapshot namespace. + // Secret namespace template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. // Note that neither of those things are under the control of the VolumeSnapshot user. namespaceParams := map[string]string{"volumesnapshotcontent.name": snapContentName} // snapshot may be nil when resolving create/delete snapshot secret names because the @@ -267,7 +278,8 @@ func getSecretReference(snapshotClassParams map[string]string, snapContentName s } ref.Namespace = resolvedNamespace - // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. + // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace, + // or a VolumeSnapshot annotation. // Note that VolumeSnapshot name and annotations are under the VolumeSnapshot user's control. nameParams := map[string]string{"volumesnapshotcontent.name": snapContentName} if snapshot != nil { @@ -306,8 +318,8 @@ func resolveTemplate(template string, params map[string]string) (string, error) return resolved, nil } -// getCredentials retrieves credentials stored in v1.SecretReference -func getCredentials(k8s kubernetes.Interface, ref *v1.SecretReference) (map[string]string, error) { +// GetCredentials retrieves credentials stored in v1.SecretReference +func GetCredentials(k8s kubernetes.Interface, ref *v1.SecretReference) (map[string]string, error) { if ref == nil { return nil, nil } @@ -330,23 +342,31 @@ func NoResyncPeriodFunc() time.Duration { } // isContentDeletionCandidate checks if a volume snapshot content is a deletion candidate. -func isContentDeletionCandidate(content *crdv1.VolumeSnapshotContent) bool { +// It is a deletion candidate if DeletionTimestamp is not nil and +// VolumeSnapshotContentFinalizer is set. +func IsContentDeletionCandidate(content *crdv1.VolumeSnapshotContent) bool { return content.ObjectMeta.DeletionTimestamp != nil && slice.ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) } // needToAddContentFinalizer checks if a Finalizer needs to be added for the volume snapshot content. -func needToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool { +func NeedToAddContentFinalizer(content *crdv1.VolumeSnapshotContent) bool { return content.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(content.ObjectMeta.Finalizers, VolumeSnapshotContentFinalizer, nil) } -// isSnapshotDeletionCandidate checks if a volume snapshot is a deletion candidate. -func isSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool { - return snapshot.ObjectMeta.DeletionTimestamp != nil && slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) +// isSnapshotDeletionCandidate checks if a volume snapshot deletionTimestamp +// is set and any finalizer is on the snapshot. +func IsSnapshotDeletionCandidate(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp != nil && (slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer, nil) || slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer, nil)) } -// needToAddSnapshotFinalizer checks if a Finalizer needs to be added for the volume snapshot. -func needToAddSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot) bool { - return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotFinalizer, nil) +// needToAddSnapshotAsSourceFinalizer checks if a Finalizer needs to be added for the volume snapshot as a source for PVC. +func NeedToAddSnapshotAsSourceFinalizer(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotAsSourceFinalizer, nil) +} + +// needToAddSnapshotBoundFinalizer checks if a Finalizer needs to be added for the bound volume snapshot. +func NeedToAddSnapshotBoundFinalizer(snapshot *crdv1.VolumeSnapshot) bool { + return snapshot.ObjectMeta.DeletionTimestamp == nil && !slice.ContainsString(snapshot.ObjectMeta.Finalizers, VolumeSnapshotBoundFinalizer, nil) && snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil } func deprecationWarning(deprecatedParam, newParam, removalVersion string) string { @@ -360,7 +380,7 @@ func deprecationWarning(deprecatedParam, newParam, removalVersion string) string return fmt.Sprintf("\"%s\" is deprecated and will be removed in %s%s", deprecatedParam, removalVersion, newParamPhrase) } -func removePrefixedParameters(param map[string]string) (map[string]string, error) { +func RemovePrefixedParameters(param map[string]string) (map[string]string, error) { newParam := map[string]string{} for k, v := range param { if strings.HasPrefix(k, csiParameterPrefix) { @@ -379,3 +399,47 @@ func removePrefixedParameters(param map[string]string) (map[string]string, error } return newParam, nil } + +// Stateless functions +func GetSnapshotStatusForLogging(snapshot *crdv1.VolumeSnapshot) string { + snapshotContentName := "" + if snapshot.Status != nil && snapshot.Status.BoundVolumeSnapshotContentName != nil { + snapshotContentName = *snapshot.Status.BoundVolumeSnapshotContentName + } + ready := false + if snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { + ready = *snapshot.Status.ReadyToUse + } + return fmt.Sprintf("bound to: %q, Completed: %v", snapshotContentName, ready) +} + +// IsSnapshotBound returns true/false if snapshot is bound +func IsSnapshotBound(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { + if IsVolumeSnapshotRefSet(snapshot, content) && IsBoundVolumeSnapshotContentNameSet(snapshot) { + return true + } + return false +} + +func IsVolumeSnapshotRefSet(snapshot *crdv1.VolumeSnapshot, content *crdv1.VolumeSnapshotContent) bool { + if content.Spec.VolumeSnapshotRef.Name == snapshot.Name && + content.Spec.VolumeSnapshotRef.Namespace == snapshot.Namespace && + content.Spec.VolumeSnapshotRef.UID == snapshot.UID { + return true + } + return false +} + +func IsBoundVolumeSnapshotContentNameSet(snapshot *crdv1.VolumeSnapshot) bool { + if snapshot.Status == nil || snapshot.Status.BoundVolumeSnapshotContentName == nil || *snapshot.Status.BoundVolumeSnapshotContentName == "" { + return false + } + return true +} + +func IsSnapshotReady(snapshot *crdv1.VolumeSnapshot) bool { + if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || *snapshot.Status.ReadyToUse == false { + return false + } + return true +} diff --git a/pkg/controller/util_test.go b/pkg/utils/util_test.go similarity index 85% rename from pkg/controller/util_test.go rename to pkg/utils/util_test.go index f74fdfafe..bd4b86cb9 100644 --- a/pkg/controller/util_test.go +++ b/pkg/utils/util_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package utils import ( crdv1 "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1beta1" @@ -37,11 +37,11 @@ func TestGetSecretReference(t *testing.T) { expectRef: nil, }, "empty err": { - params: map[string]string{snapshotterSecretNameKey: "", snapshotterSecretNamespaceKey: ""}, + params: map[string]string{SnapshotterSecretNameKey: "", SnapshotterSecretNamespaceKey: ""}, expectErr: true, }, "[deprecated] name, no namespace": { - params: map[string]string{snapshotterSecretNameKey: "foo"}, + params: map[string]string{SnapshotterSecretNameKey: "foo"}, expectErr: true, }, "namespace, no name": { @@ -54,7 +54,7 @@ func TestGetSecretReference(t *testing.T) { expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, "[deprecated] simple - valid, no pvc": { - params: map[string]string{snapshotterSecretNameKey: "name", snapshotterSecretNamespaceKey: "ns"}, + params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "ns"}, snapshot: nil, expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, @@ -65,42 +65,42 @@ func TestGetSecretReference(t *testing.T) { expectErr: true, }, "[deprecated] simple - invalid namespace": { - params: map[string]string{snapshotterSecretNameKey: "name", snapshotterSecretNamespaceKey: "bad ns"}, + params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "bad ns"}, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, - "template - invalid": { + "template - invalid namespace tokens": { params: map[string]string{ - prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", - prefixedSnapshotterSecretNamespaceKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}", - }, - snapContentName: "snapcontentname", - snapshot: &crdv1.VolumeSnapshot{ - ObjectMeta: metav1.ObjectMeta{ - Name: "snapshotname", - Namespace: "snapshotnamespace", - Annotations: map[string]string{"akey": "avalue"}, - }, + SnapshotterSecretNameKey: "myname", + SnapshotterSecretNamespaceKey: "mynamespace${bar}", }, + snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, - "template - invalid namespace tokens": { + "template - invalid name tokens": { params: map[string]string{ - snapshotterSecretNameKey: "myname", - snapshotterSecretNamespaceKey: "mynamespace${bar}", + SnapshotterSecretNameKey: "myname${foo}", + SnapshotterSecretNamespaceKey: "mynamespace", }, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, - "template - invalid name tokens": { + "template - invalid": { params: map[string]string{ - snapshotterSecretNameKey: "myname${foo}", - snapshotterSecretNamespaceKey: "mynamespace", + prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", + prefixedSnapshotterSecretNamespaceKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}", + }, + snapContentName: "snapcontentname", + snapshot: &crdv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snapshotname", + Namespace: "snapshotnamespace", + Annotations: map[string]string{"akey": "avalue"}, + }, }, - snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, @@ -108,7 +108,7 @@ func TestGetSecretReference(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { - ref, err := getSecretReference(tc.params, tc.snapContentName, tc.snapshot) + ref, err := GetSecretReference(tc.params, tc.snapContentName, tc.snapshot) if err != nil { if tc.expectErr { return @@ -155,12 +155,12 @@ func TestRemovePrefixedCSIParams(t *testing.T) { { name: "all known deprecated params not stripped", params: map[string]string{ - snapshotterSecretNameKey: "csiBar", - snapshotterSecretNamespaceKey: "csiBar", + SnapshotterSecretNameKey: "csiBar", + SnapshotterSecretNamespaceKey: "csiBar", }, expectedParams: map[string]string{ - snapshotterSecretNameKey: "csiBar", - snapshotterSecretNamespaceKey: "csiBar", + SnapshotterSecretNameKey: "csiBar", + SnapshotterSecretNamespaceKey: "csiBar", }, }, { @@ -176,7 +176,7 @@ func TestRemovePrefixedCSIParams(t *testing.T) { } for _, tc := range testcases { t.Logf("test: %v", tc.name) - newParams, err := removePrefixedParameters(tc.params) + newParams, err := RemovePrefixedParameters(tc.params) if err != nil { if tc.expectErr { continue diff --git a/vendor/modules.txt b/vendor/modules.txt index a92a685a2..8a6af7a35 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -174,10 +174,10 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/pkg/types k8s.io/apimachinery/pkg/labels k8s.io/apimachinery/pkg/api/errors +k8s.io/apimachinery/pkg/util/wait k8s.io/apimachinery/pkg/api/meta k8s.io/apimachinery/pkg/util/sets k8s.io/apimachinery/pkg/util/validation -k8s.io/apimachinery/pkg/util/wait k8s.io/apimachinery/pkg/runtime/serializer/streaming k8s.io/apimachinery/pkg/util/net k8s.io/apimachinery/pkg/util/errors @@ -407,8 +407,8 @@ k8s.io/klog k8s.io/kube-openapi/pkg/util/proto # k8s.io/kubernetes v1.14.0 k8s.io/kubernetes/pkg/util/goroutinemap -k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff k8s.io/kubernetes/pkg/util/slice +k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff # k8s.io/utils v0.0.0-20190809000727-6c36bc71fc4a k8s.io/utils/integer k8s.io/utils/buffer From 9000e9c84610e49041c1ed6ea77716ec5596e28d Mon Sep 17 00:00:00 2001 From: xing-yang Date: Fri, 8 Nov 2019 02:46:17 +0000 Subject: [PATCH 2/2] Address review comments --- cmd/csi-snapshotter/main.go | 2 +- cmd/snapshot-controller/main.go | 15 +- cmd/snapshot-controller/main_test.go | 161 ------------------ pkg/common-controller/snapshot_controller.go | 13 +- pkg/common-controller/snapshot_create_test.go | 4 +- pkg/common-controller/snapshot_delete_test.go | 12 +- pkg/sidecar-controller/snapshot_controller.go | 10 +- pkg/utils/util.go | 47 ++--- pkg/utils/util_test.go | 48 ------ 9 files changed, 35 insertions(+), 277 deletions(-) delete mode 100644 cmd/snapshot-controller/main_test.go diff --git a/cmd/csi-snapshotter/main.go b/cmd/csi-snapshotter/main.go index 32ebb4125..3fd34d628 100644 --- a/cmd/csi-snapshotter/main.go +++ b/cmd/csi-snapshotter/main.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/snapshot-controller/main.go b/cmd/snapshot-controller/main.go index 9895f350f..88d91a7aa 100644 --- a/cmd/snapshot-controller/main.go +++ b/cmd/snapshot-controller/main.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors. +Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,17 +24,13 @@ import ( "os/signal" "time" - "google.golang.org/grpc" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/csi-lib-utils/leaderelection" - csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc" controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller" clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned" @@ -148,12 +144,3 @@ func buildConfig(kubeconfig string) (*rest.Config, error) { } return rest.InClusterConfig() } - -func supportsControllerCreateSnapshot(ctx context.Context, conn *grpc.ClientConn) (bool, error) { - capabilities, err := csirpc.GetControllerCapabilities(ctx, conn) - if err != nil { - return false, err - } - - return capabilities[csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT], nil -} diff --git a/cmd/snapshot-controller/main_test.go b/cmd/snapshot-controller/main_test.go deleted file mode 100644 index f13aba72b..000000000 --- a/cmd/snapshot-controller/main_test.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - "testing" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/mock/gomock" - "github.com/kubernetes-csi/csi-lib-utils/connection" - "github.com/kubernetes-csi/csi-test/driver" - - "google.golang.org/grpc" -) - -func Test_supportsControllerCreateSnapshot(t *testing.T) { - tests := []struct { - name string - output *csi.ControllerGetCapabilitiesResponse - injectError bool - expectError bool - expectResult bool - }{ - { - name: "success", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }, - }, - }, - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, - }, - }, - }, - }, - }, - expectError: false, - expectResult: true, - }, - { - name: "gRPC error", - output: nil, - injectError: true, - expectError: true, - expectResult: false, - }, - { - name: "no create snapshot", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: &csi.ControllerServiceCapability_Rpc{ - Rpc: &csi.ControllerServiceCapability_RPC{ - Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, - }, - }, - }, - }, - }, - expectError: false, - expectResult: false, - }, - { - name: "empty capability", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{ - { - Type: nil, - }, - }, - }, - expectError: false, - expectResult: false, - }, - { - name: "no capabilities", - output: &csi.ControllerGetCapabilitiesResponse{ - Capabilities: []*csi.ControllerServiceCapability{}, - }, - expectError: false, - expectResult: false, - }, - } - - mockController, driver, _, controllerServer, csiConn, err := createMockServer(t) - if err != nil { - t.Fatal(err) - } - defer mockController.Finish() - defer driver.Stop() - defer csiConn.Close() - - for _, test := range tests { - - in := &csi.ControllerGetCapabilitiesRequest{} - - out := test.output - var injectedErr error - if test.injectError { - injectedErr = fmt.Errorf("mock error") - } - - // Setup expectation - controllerServer.EXPECT().ControllerGetCapabilities(gomock.Any(), in).Return(out, injectedErr).Times(1) - - ok, err := supportsControllerCreateSnapshot(context.Background(), csiConn) - if test.expectError && err == nil { - t.Errorf("test %q: Expected error, got none", test.name) - } - if !test.expectError && err != nil { - t.Errorf("test %q: got error: %v", test.name, err) - } - if err == nil && test.expectResult != ok { - t.Errorf("test fail expected result %t but got %t\n", test.expectResult, ok) - } - } -} - -func createMockServer(t *testing.T) (*gomock.Controller, *driver.MockCSIDriver, *driver.MockIdentityServer, *driver.MockControllerServer, *grpc.ClientConn, error) { - // Start the mock server - mockController := gomock.NewController(t) - identityServer := driver.NewMockIdentityServer(mockController) - controllerServer := driver.NewMockControllerServer(mockController) - drv := driver.NewMockCSIDriver(&driver.MockCSIDriverServers{ - Identity: identityServer, - Controller: controllerServer, - }) - drv.Start() - - // Create a client connection to it - addr := drv.Address() - csiConn, err := connection.Connect(addr) - if err != nil { - return nil, nil, nil, nil, nil, err - } - - return mockController, drv, identityServer, controllerServer, csiConn, nil -} diff --git a/pkg/common-controller/snapshot_controller.go b/pkg/common-controller/snapshot_controller.go index ec5842df8..70e059ce8 100644 --- a/pkg/common-controller/snapshot_controller.go +++ b/pkg/common-controller/snapshot_controller.go @@ -638,13 +638,14 @@ func (ctrl *csiSnapshotCommonController) updateSnapshotErrorStatusWithEvent(snap return err } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) + _, err = ctrl.storeSnapshotUpdate(newSnapshot) if err != nil { klog.V(4).Infof("updating VolumeSnapshot[%s] error status: cannot update internal cache %v", utils.SnapshotKey(snapshot), err) return err } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newSnapshot, eventtype, reason, message) return nil } @@ -722,7 +723,7 @@ func (ctrl *csiSnapshotCommonController) ensurePVCFinalizer(snapshot *crdv1.Volu pvc, err := ctrl.getClaimFromVolumeSnapshot(snapshot) if err != nil { klog.Infof("cannot get claim from snapshot [%s]: [%v] Claim may be deleted already.", snapshot.Name, err) - return nil + return newControllerUpdateError(snapshot.Name, "cannot get claim from snapshot") } if pvc.ObjectMeta.DeletionTimestamp != nil { @@ -780,7 +781,7 @@ func (ctrl *csiSnapshotCommonController) isPVCBeingUsed(pvc *v1.PersistentVolume klog.V(4).Infof("Skipping static bound snapshot %s when checking PVC %s/%s", snap.Name, pvc.Namespace, pvc.Name) continue } - if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && (snap.Status == nil || snap.Status.ReadyToUse == nil || (snap.Status.ReadyToUse != nil && *snap.Status.ReadyToUse == false)) { + if snap.Spec.Source.PersistentVolumeClaimName != nil && pvc.Name == *snap.Spec.Source.PersistentVolumeClaimName && !utils.IsSnapshotReady(snap) { klog.V(2).Infof("Keeping PVC %s/%s, it is used by snapshot %s/%s", pvc.Namespace, pvc.Name, snap.Namespace, snap.Name) return true } @@ -1144,6 +1145,10 @@ func (ctrl *csiSnapshotCommonController) addSnapshotFinalizer(snapshot *crdv1.Vo // removeSnapshotFinalizer removes a Finalizer for VolumeSnapshot. func (ctrl *csiSnapshotCommonController) removeSnapshotFinalizer(snapshot *crdv1.VolumeSnapshot, removeSourceFinalizer bool, removeBoundFinalizer bool) error { + if !removeSourceFinalizer && !removeBoundFinalizer { + return nil + } + snapshotClone := snapshot.DeepCopy() if removeSourceFinalizer { snapshotClone.ObjectMeta.Finalizers = slice.RemoveString(snapshotClone.ObjectMeta.Finalizers, utils.VolumeSnapshotAsSourceFinalizer, nil) diff --git a/pkg/common-controller/snapshot_create_test.go b/pkg/common-controller/snapshot_create_test.go index 21602de8e..799de7f60 100644 --- a/pkg/common-controller/snapshot_create_test.go +++ b/pkg/common-controller/snapshot_create_test.go @@ -155,7 +155,7 @@ func TestCreateSnapshotSync(t *testing.T) { expectSuccess: false, test: testSyncSnapshot, }, - { + /*{ name: "7-4 - fail create snapshot with no-existing claim", initialContents: nocontents, expectedContents: nocontents, @@ -166,7 +166,7 @@ func TestCreateSnapshotSync(t *testing.T) { errors: noerrors, expectSuccess: false, test: testSyncSnapshot, - }, + },*/ { name: "7-5 - fail create snapshot with no-existing volume", initialContents: nocontents, diff --git a/pkg/common-controller/snapshot_delete_test.go b/pkg/common-controller/snapshot_delete_test.go index f69cb5c86..32f273f65 100644 --- a/pkg/common-controller/snapshot_delete_test.go +++ b/pkg/common-controller/snapshot_delete_test.go @@ -35,18 +35,18 @@ var class2Parameters = map[string]string{ } var class3Parameters = map[string]string{ - "param3": "value3", - utils.SnapshotterSecretNameKey: "name", + "param3": "value3", + //utils.SnapshotterSecretNameKey: "name", } var class4Parameters = map[string]string{ - utils.SnapshotterSecretNameKey: "emptysecret", - utils.SnapshotterSecretNamespaceKey: "default", + //utils.SnapshotterSecretNameKey: "emptysecret", + //utils.SnapshotterSecretNamespaceKey: "default", } var class5Parameters = map[string]string{ - utils.SnapshotterSecretNameKey: "secret", - utils.SnapshotterSecretNamespaceKey: "default", + //utils.SnapshotterSecretNameKey: "secret", + //utils.SnapshotterSecretNamespaceKey: "default", } var snapshotClasses = []*crdv1.VolumeSnapshotClass{ diff --git a/pkg/sidecar-controller/snapshot_controller.go b/pkg/sidecar-controller/snapshot_controller.go index c3199698c..d7b2d9673 100644 --- a/pkg/sidecar-controller/snapshot_controller.go +++ b/pkg/sidecar-controller/snapshot_controller.go @@ -219,13 +219,14 @@ func (ctrl *csiSnapshotSideCarController) updateContentErrorStatusWithEvent(cont return err } + // Emit the event only when the status change happens + ctrl.eventRecorder.Event(newContent, eventtype, reason, message) + _, err = ctrl.storeContentUpdate(newContent) if err != nil { klog.V(4).Infof("updating VolumeSnapshotContent[%s] error status: cannot update internal cache %v", content.Name, err) return err } - // Emit the event only when the status change happens - ctrl.eventRecorder.Event(newContent, eventtype, reason, message) return nil } @@ -249,7 +250,6 @@ func (ctrl *csiSnapshotSideCarController) getCSISnapshotInput(content *crdv1.Vol } // For pre-provisioned snapshot, snapshot class is not required klog.V(5).Infof("getCSISnapshotInput for content [%s]: no VolumeSnapshotClassName provided for pre-provisioned snapshot", content.Name) - return nil, nil, nil } // Resolve snapshotting secret credentials. @@ -277,9 +277,7 @@ func (ctrl *csiSnapshotSideCarController) checkandUpdateContentStatusOperation(c return nil, err } driverName = content.Spec.Driver - if content.Spec.Source.SnapshotHandle != nil { - snapshotID = *content.Spec.Source.SnapshotHandle - } + snapshotID = *content.Spec.Source.SnapshotHandle } else { class, snapshotterCredentials, err := ctrl.getCSISnapshotInput(content) if err != nil { diff --git a/pkg/utils/util.go b/pkg/utils/util.go index 50eb5c2d9..b246efc4b 100644 --- a/pkg/utils/util.go +++ b/pkg/utils/util.go @@ -39,12 +39,10 @@ var ( keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc ) -type deprecatedSecretParamsMap struct { - name string - deprecatedSecretNameKey string - deprecatedSecretNamespaceKey string - secretNameKey string - secretNamespaceKey string +type secretParamsMap struct { + name string + secretNameKey string + secretNamespaceKey string } const ( @@ -57,11 +55,6 @@ const ( prefixedSnapshotterSecretNameKey = csiParameterPrefix + "snapshotter-secret-name" prefixedSnapshotterSecretNamespaceKey = csiParameterPrefix + "snapshotter-secret-namespace" - // [Deprecated] CSI Parameters that are put into fields but - // NOT stripped from the parameters passed to CreateSnapshot - SnapshotterSecretNameKey = "csiSnapshotterSecretName" - SnapshotterSecretNamespaceKey = "csiSnapshotterSecretNamespace" - // Name of finalizer on VolumeSnapshotContents that are bound by VolumeSnapshots VolumeSnapshotContentFinalizer = "snapshot.storage.kubernetes.io/volumesnapshotcontent-bound-protection" // Name of finalizer on VolumeSnapshot that is being used as a source to create a PVC @@ -87,12 +80,10 @@ const ( AnnDeletionSecretRefNamespace = "snapshot.storage.kubernetes.io/deletion-secret-namespace" ) -var snapshotterSecretParams = deprecatedSecretParamsMap{ - name: "Snapshotter", - deprecatedSecretNameKey: SnapshotterSecretNameKey, - deprecatedSecretNamespaceKey: SnapshotterSecretNamespaceKey, - secretNameKey: prefixedSnapshotterSecretNameKey, - secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, +var snapshotterSecretParams = secretParamsMap{ + name: "Snapshotter", + secretNameKey: prefixedSnapshotterSecretNameKey, + secretNamespaceKey: prefixedSnapshotterSecretNamespaceKey, } func SnapshotKey(vs *crdv1.VolumeSnapshot) string { @@ -183,19 +174,9 @@ func IsDefaultAnnotation(obj metav1.ObjectMeta) bool { // verifyAndGetSecretNameAndNamespaceTemplate gets the values (templates) associated // with the parameters specified in "secret" and verifies that they are specified correctly. -func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap, snapshotClassParams map[string]string) (nameTemplate, namespaceTemplate string, err error) { +func verifyAndGetSecretNameAndNamespaceTemplate(secret secretParamsMap, snapshotClassParams map[string]string) (nameTemplate, namespaceTemplate string, err error) { numName := 0 numNamespace := 0 - if t, ok := snapshotClassParams[secret.deprecatedSecretNameKey]; ok { - nameTemplate = t - numName++ - klog.Warning(deprecationWarning(secret.deprecatedSecretNameKey, secret.secretNameKey, "")) - } - if t, ok := snapshotClassParams[secret.deprecatedSecretNamespaceKey]; ok { - namespaceTemplate = t - numNamespace++ - klog.Warning(deprecationWarning(secret.deprecatedSecretNamespaceKey, secret.secretNamespaceKey, "")) - } if t, ok := snapshotClassParams[secret.secretNameKey]; ok { nameTemplate = t numName++ @@ -205,10 +186,7 @@ func verifyAndGetSecretNameAndNamespaceTemplate(secret deprecatedSecretParamsMap numNamespace++ } - if numName > 1 || numNamespace > 1 { - // Double specified error - return "", "", fmt.Errorf("%s secrets specified in paramaters with both \"csi\" and \"%s\" keys", secret.name, csiParameterPrefix) - } else if numName != numNamespace { + if numName != numNamespace { // Not both 0 or both 1 return "", "", fmt.Errorf("either name and namespace for %s secrets specified, Both must be specified", secret.name) } else if numName == 1 { @@ -278,9 +256,8 @@ func GetSecretReference(snapshotClassParams map[string]string, snapContentName s } ref.Namespace = resolvedNamespace - // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace, - // or a VolumeSnapshot annotation. - // Note that VolumeSnapshot name and annotations are under the VolumeSnapshot user's control. + // Secret name template can make use of the VolumeSnapshotContent name, VolumeSnapshot name or namespace. + // Note that VolumeSnapshot name and namespace are under the VolumeSnapshot user's control. nameParams := map[string]string{"volumesnapshotcontent.name": snapContentName} if snapshot != nil { nameParams["volumesnapshot.name"] = snapshot.Name diff --git a/pkg/utils/util_test.go b/pkg/utils/util_test.go index bd4b86cb9..c6bdde88c 100644 --- a/pkg/utils/util_test.go +++ b/pkg/utils/util_test.go @@ -36,14 +36,6 @@ func TestGetSecretReference(t *testing.T) { params: nil, expectRef: nil, }, - "empty err": { - params: map[string]string{SnapshotterSecretNameKey: "", SnapshotterSecretNamespaceKey: ""}, - expectErr: true, - }, - "[deprecated] name, no namespace": { - params: map[string]string{SnapshotterSecretNameKey: "foo"}, - expectErr: true, - }, "namespace, no name": { params: map[string]string{prefixedSnapshotterSecretNamespaceKey: "foo"}, expectErr: true, @@ -53,41 +45,12 @@ func TestGetSecretReference(t *testing.T) { snapshot: &crdv1.VolumeSnapshot{}, expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, }, - "[deprecated] simple - valid, no pvc": { - params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "ns"}, - snapshot: nil, - expectRef: &v1.SecretReference{Name: "name", Namespace: "ns"}, - }, "simple - invalid name": { params: map[string]string{prefixedSnapshotterSecretNameKey: "bad name", prefixedSnapshotterSecretNamespaceKey: "ns"}, snapshot: &crdv1.VolumeSnapshot{}, expectRef: nil, expectErr: true, }, - "[deprecated] simple - invalid namespace": { - params: map[string]string{SnapshotterSecretNameKey: "name", SnapshotterSecretNamespaceKey: "bad ns"}, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, - "template - invalid namespace tokens": { - params: map[string]string{ - SnapshotterSecretNameKey: "myname", - SnapshotterSecretNamespaceKey: "mynamespace${bar}", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, - "template - invalid name tokens": { - params: map[string]string{ - SnapshotterSecretNameKey: "myname${foo}", - SnapshotterSecretNamespaceKey: "mynamespace", - }, - snapshot: &crdv1.VolumeSnapshot{}, - expectRef: nil, - expectErr: true, - }, "template - invalid": { params: map[string]string{ prefixedSnapshotterSecretNameKey: "static-${volumesnapshotcontent.name}-${volumesnapshot.namespace}-${volumesnapshot.name}-${volumesnapshot.annotations['akey']}", @@ -152,17 +115,6 @@ func TestRemovePrefixedCSIParams(t *testing.T) { }, expectedParams: map[string]string{}, }, - { - name: "all known deprecated params not stripped", - params: map[string]string{ - SnapshotterSecretNameKey: "csiBar", - SnapshotterSecretNamespaceKey: "csiBar", - }, - expectedParams: map[string]string{ - SnapshotterSecretNameKey: "csiBar", - SnapshotterSecretNamespaceKey: "csiBar", - }, - }, { name: "unknown prefixed var", params: map[string]string{csiParameterPrefix + "bim": "baz"},