diff --git a/changelogs/unreleased/270-pawanpraka1 b/changelogs/unreleased/270-pawanpraka1 new file mode 100644 index 000000000..6bbcd40e9 --- /dev/null +++ b/changelogs/unreleased/270-pawanpraka1 @@ -0,0 +1 @@ +try volume creation on all the nodes that satisfy topology contraints diff --git a/deploy/yamls/zfsvolume-crd.yaml b/deploy/yamls/zfsvolume-crd.yaml index 914adf34a..38934b45e 100644 --- a/deploy/yamls/zfsvolume-crd.yaml +++ b/deploy/yamls/zfsvolume-crd.yaml @@ -231,6 +231,7 @@ spec: enum: - Pending - Ready + - Failed type: string type: object required: diff --git a/deploy/zfs-operator.yaml b/deploy/zfs-operator.yaml index 86eccbac7..cbe4dddb6 100644 --- a/deploy/zfs-operator.yaml +++ b/deploy/zfs-operator.yaml @@ -252,6 +252,7 @@ spec: enum: - Pending - Ready + - Failed type: string type: object required: diff --git a/go.sum b/go.sum index 7126bfad5..3f9da53f6 100644 --- a/go.sum +++ b/go.sum @@ -317,7 +317,6 @@ github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7ir github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -352,7 +351,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -361,7 +359,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -379,7 +376,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= @@ -398,13 +394,11 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -423,7 +417,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -481,7 +474,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2l golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= @@ -529,14 +521,12 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= @@ -563,7 +553,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -612,7 +601,6 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9 h1:6XzpBoANz1NqMNfDXzc2QmHmbb1vyMsvRfoP5rM+K1I= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -646,7 +634,6 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -717,7 +704,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhY sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/testing_frameworks v0.1.1 h1:cP2l8fkA3O9vekpy5Ks8mmA0NW/F7yBdXf8brkWhVrs= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/apis/openebs.io/zfs/v1/zfsvolume.go b/pkg/apis/openebs.io/zfs/v1/zfsvolume.go index a93bff170..86dff5bad 100644 --- a/pkg/apis/openebs.io/zfs/v1/zfsvolume.go +++ b/pkg/apis/openebs.io/zfs/v1/zfsvolume.go @@ -186,6 +186,6 @@ type VolStatus struct { // The state "Pending" means that the volume creation request has not // processed yet. The state "Ready" means that the volume has been created // and it is ready for the use. - // +kubebuilder:validation:Enum=Pending;Ready + // +kubebuilder:validation:Enum=Pending;Ready;Failed State string `json:"state,omitempty"` } diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index bb803d611..8d9268035 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -101,23 +101,6 @@ func getRoundedCapacity(size int64) int64 { return ((size + Mi - 1) / Mi) * Mi } -func waitForReadyVolume(volname string) error { - for true { - vol, err := zfs.GetZFSVolume(volname) - if err != nil { - return status.Errorf(codes.Internal, - "zfs: wait failed, not able to get the volume %s %s", volname, err.Error()) - } - - switch vol.Status.State { - case zfs.ZFSStatusReady: - return nil - } - time.Sleep(time.Second) - } - return nil -} - func waitForVolDestroy(volname string) error { for true { _, err := zfs.GetZFSVolume(volname) @@ -129,6 +112,7 @@ func waitForVolDestroy(volname string) error { "zfs: destroy wait failed, not able to get the volume %s %s", volname, err.Error()) } time.Sleep(time.Second) + klog.Infof("waiting for volume to be destroyed %s", volname) } return nil } @@ -151,7 +135,7 @@ func waitForReadySnapshot(snapname string) error { } // CreateZFSVolume create new zfs volume from csi volume request -func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) { +func CreateZFSVolume(ctx context.Context, req *csi.CreateVolumeRequest) (string, error) { volName := strings.ToLower(req.GetName()) size := getRoundedCapacity(req.GetCapacityRange().RequiredBytes) @@ -191,6 +175,10 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) { return "", status.Errorf(codes.AlreadyExists, "volume %s already present", volName) } + if vol.Status.State != zfs.ZFSStatusReady { + return "", status.Errorf(codes.Aborted, + "volume %s request already pending", volName) + } return vol.Spec.OwnerNodeID, nil } } @@ -200,21 +188,19 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) { return "", status.Errorf(codes.Internal, "get node map failed : %s", err.Error()) } - // run the scheduler get the preferred nodelist - var selected string - nodelist := schd.Scheduler(req, nmap) - if len(nodelist) != 0 { - selected = nodelist[0] - } - if len(selected) == 0 { + var prfList []string + + if node, ok := parameters["node"]; ok { // (hack): CSI Sanity test does not pass topology information - selected = parameters["node"] - if len(selected) == 0 { - return "", status.Error(codes.Internal, "scheduler failed, not able to select a node to create the PV") - } + prfList = append(prfList, node) + } else { + // run the scheduler + prfList = schd.Scheduler(req, nmap) } - klog.Infof("scheduled the volume %s/%s on node %s", pool, volName, selected) + if len(prfList) == 0 { + return "", status.Error(codes.Internal, "scheduler failed, node list is empty for creating the PV") + } volObj, err := volbuilder.NewBuilder(). WithName(volName). @@ -227,7 +213,6 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) { WithKeyFormat(kf). WithKeyLocation(kl). WithThinProv(tp). - WithOwnerNode(selected). WithVolumeType(vtype). WithVolumeStatus(zfs.ZFSStatusPending). WithFsType(fstype). @@ -238,17 +223,36 @@ func CreateZFSVolume(req *csi.CreateVolumeRequest) (string, error) { return "", status.Error(codes.Internal, err.Error()) } - err = zfs.ProvisionVolume(volObj) + klog.Infof("zfs: trying volume creation %s/%s on node %s", pool, volName, prfList) + + // try volume creation sequentially on all nodes + for _, node := range prfList { + vol, _ := volbuilder.BuildFrom(volObj).WithOwnerNode(node).WithVolumeStatus(zfs.ZFSStatusPending).Build() + + timeout := false + + timeout, err = zfs.ProvisionVolume(ctx, vol) + if err == nil { + return node, nil + } + + // if timeout reached, return the error and let csi retry the volume creation + if timeout { + break + } + } + if err != nil { - return "", status.Errorf(codes.Internal, - "not able to provision the volume %s", err.Error()) + // volume provisioning failed, delete the zfs volume resource + zfs.DeleteVolume(volName) // ignore error } - return selected, nil + return "", status.Errorf(codes.Internal, + "not able to provision the volume, nodes %v, err : %s", prfList, err.Error()) } // CreateVolClone creates the clone from a volume -func CreateVolClone(req *csi.CreateVolumeRequest, srcVol string) (string, error) { +func CreateVolClone(ctx context.Context, req *csi.CreateVolumeRequest, srcVol string) (string, error) { volName := strings.ToLower(req.GetName()) parameters := req.GetParameters() // lower case keys, cf CreateZFSVolume() @@ -286,17 +290,17 @@ func CreateVolClone(req *csi.CreateVolumeRequest, srcVol string) (string, error) // use the snapshot name same as new volname volObj.Spec.SnapName = vol.Name + "@" + volName - err = zfs.ProvisionVolume(volObj) + _, err = zfs.ProvisionVolume(ctx, volObj) if err != nil { return "", status.Errorf(codes.Internal, - "clone: not able to provision the volume %s", err.Error()) + "clone: not able to provision the volume err : %s", err.Error()) } return selected, nil } // CreateSnapClone creates the clone from a snapshot -func CreateSnapClone(req *csi.CreateVolumeRequest, snapshot string) (string, error) { +func CreateSnapClone(ctx context.Context, req *csi.CreateVolumeRequest, snapshot string) (string, error) { volName := strings.ToLower(req.GetName()) parameters := req.GetParameters() // lower case keys, cf CreateZFSVolume() @@ -339,10 +343,10 @@ func CreateSnapClone(req *csi.CreateVolumeRequest, snapshot string) (string, err volObj.Spec = snap.Spec volObj.Spec.SnapName = strings.ToLower(snapshot) - err = zfs.ProvisionVolume(volObj) + _, err = zfs.ProvisionVolume(ctx, volObj) if err != nil { return "", status.Errorf(codes.Internal, - "not able to provision the clone volume %s", err.Error()) + "not able to provision the clone volume err : %s", err.Error()) } return selected, nil @@ -372,23 +376,19 @@ func (cs *controller) CreateVolume( if contentSource != nil && contentSource.GetSnapshot() != nil { snapshotID := contentSource.GetSnapshot().GetSnapshotId() - selected, err = CreateSnapClone(req, snapshotID) + selected, err = CreateSnapClone(ctx, req, snapshotID) } else if contentSource != nil && contentSource.GetVolume() != nil { srcVol := contentSource.GetVolume().GetVolumeId() - selected, err = CreateVolClone(req, srcVol) + selected, err = CreateVolClone(ctx, req, srcVol) } else { - selected, err = CreateZFSVolume(req) + selected, err = CreateZFSVolume(ctx, req) } if err != nil { return nil, err } - if _, ok := parameters["wait"]; ok { - if err := waitForReadyVolume(volName); err != nil { - return nil, err - } - } + klog.Infof("created the volume %s/%s on node %s", pool, volName, selected) sendEventOrIgnore(pvcName, volName, strconv.FormatInt(int64(size), 10), "zfs-localpv", analytics.VolumeProvision) @@ -411,11 +411,7 @@ func (cs *controller) DeleteVolume( klog.Infof("received request to delete volume {%s}", req.VolumeId) - var ( - err error - ) - - if err = cs.validateDeleteVolumeReq(req); err != nil { + if err := cs.validateDeleteVolumeReq(req); err != nil { return nil, err } @@ -438,6 +434,11 @@ func (cs *controller) DeleteVolume( ) } + // if volume is not ready, create volume will delete it + if vol.Status.State != zfs.ZFSStatusReady { + return nil, status.Error(codes.Internal, "can not delete, volume creation is in progress") + } + // Delete the corresponding ZV CR err = zfs.DeleteVolume(volumeID) if err != nil { diff --git a/pkg/mgmt/volume/volume.go b/pkg/mgmt/volume/volume.go index 0814def42..0edc219fb 100644 --- a/pkg/mgmt/volume/volume.go +++ b/pkg/mgmt/volume/volume.go @@ -95,7 +95,9 @@ func (c *ZVController) syncZV(zv *apis.ZFSVolume) error { err = zfs.CreateVolume(zv) } if err == nil { - err = zfs.UpdateZvolInfo(zv) + err = zfs.UpdateZvolInfo(zv, zfs.ZFSStatusReady) + } else { + err = zfs.UpdateZvolInfo(zv, zfs.ZFSStatusFailed) } } } @@ -132,7 +134,8 @@ func (c *ZVController) updateZV(oldObj, newObj interface{}) { oldZV, _ := oldObj.(*apis.ZFSVolume) if zfs.PropertyChanged(oldZV, newZV) || - c.isDeletionCandidate(newZV) { + c.isDeletionCandidate(newZV) || + newZV.Status.State == zfs.ZFSStatusPending { klog.Infof("Got update event for ZV %s/%s", newZV.Spec.PoolName, newZV.Name) c.enqueueZV(newZV) } diff --git a/pkg/zfs/volume.go b/pkg/zfs/volume.go index cb80ac330..27ec35089 100644 --- a/pkg/zfs/volume.go +++ b/pkg/zfs/volume.go @@ -15,8 +15,11 @@ package zfs import ( + "context" + "fmt" "os" "strconv" + "time" apis "github.com/openebs/zfs-localpv/pkg/apis/openebs.io/zfs/v1" "github.com/openebs/zfs-localpv/pkg/builder/bkpbuilder" @@ -80,18 +83,63 @@ func init() { GoogleAnalyticsEnabled = os.Getenv(GoogleAnalyticsKey) } +func checkVolCreation(ctx context.Context, volname string) (bool, error) { + timeout := time.After(10 * time.Second) + for { + select { + case <-ctx.Done(): + return true, fmt.Errorf("zfs: context deadline reached") + case <-timeout: + return true, fmt.Errorf("zfs: vol creation timeout reached") + default: + vol, err := GetZFSVolume(volname) + if err != nil { + return false, fmt.Errorf("zfs: wait failed, not able to get the volume %s %s", volname, err.Error()) + } + + switch vol.Status.State { + case ZFSStatusReady: + return false, nil + case ZFSStatusFailed: + return false, fmt.Errorf("zfs: volume creation failed") + } + + klog.Infof("zfs: waiting for volume %s/%s to be created on node %s", + vol.Spec.PoolName, volname, vol.Spec.OwnerNodeID) + + time.Sleep(time.Second) + } + } +} + // ProvisionVolume creates a ZFSVolume(zv) CR, // watcher for zvc is present in CSI agent func ProvisionVolume( + ctx context.Context, vol *apis.ZFSVolume, -) error { +) (bool, error) { + timeout := false + zv, err := GetZFSVolume(vol.Name) - _, err := volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Create(vol) if err == nil { - klog.Infof("provisioned volume %s", vol.Name) + // update the spec and status + zv.Spec = vol.Spec + zv.Status = vol.Status + _, err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Update(zv) + } else { + _, err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Create(vol) } - return err + if err == nil { + timeout, err = checkVolCreation(ctx, vol.Name) + } + + if err != nil { + klog.Infof("zfs: volume %s/%s provisioning failed on node %s err: %s", + vol.Spec.PoolName, vol.Name, vol.Spec.OwnerNodeID, err.Error()) + } + + return timeout, err } // ResizeVolume resizes the zfs volume @@ -138,7 +186,9 @@ func GetVolume(volumeID string) (*apis.ZFSVolume, error) { func DeleteVolume(volumeID string) (err error) { err = volbuilder.NewKubeclient().WithNamespace(OpenEBSNamespace).Delete(volumeID) if err == nil { - klog.Infof("deprovisioned volume %s", volumeID) + klog.Infof("zfs: deleted the volume %s", volumeID) + } else { + klog.Infof("zfs: volume %s deletion failed %s", volumeID, err.Error()) } return @@ -179,17 +229,22 @@ func GetZFSVolumeState(volID string) (string, string, error) { } // UpdateZvolInfo updates ZFSVolume CR with node id and finalizer -func UpdateZvolInfo(vol *apis.ZFSVolume) error { - finalizers := []string{ZFSFinalizer} +func UpdateZvolInfo(vol *apis.ZFSVolume, status string) error { + finalizers := []string{} labels := map[string]string{ZFSNodeKey: NodeID} if vol.Finalizers != nil { return nil } + switch status { + case ZFSStatusReady: + finalizers = append(finalizers, ZFSFinalizer) + } + newVol, err := volbuilder.BuildFrom(vol). WithFinalizer(finalizers). - WithVolumeStatus(ZFSStatusReady). + WithVolumeStatus(status). WithLabels(labels).Build() if err != nil {