From 9369089a1e2c1735e9c9811754c7b678f8fec156 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Mon, 9 Sep 2019 11:41:06 -0600 Subject: [PATCH 1/2] velero install: wait for restic daemonset to be ready Signed-off-by: Steve Kriss --- pkg/cmd/cli/install/install.go | 9 ++++++- pkg/install/install.go | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 190dde93b4..53226d1c45 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -238,10 +238,17 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { } if o.Wait { - fmt.Println("Waiting for Velero to be ready.") + fmt.Println("Waiting for Velero deployment to be ready.") if _, err = install.DeploymentIsReady(factory, o.Namespace); err != nil { return errors.Wrap(err, errorMsg) } + + if o.UseRestic { + fmt.Println("Waiting for Velero restic daemonset to be ready.") + if _, err = install.DaemonSetIsReady(factory, o.Namespace); err != nil { + return errors.Wrap(err, errorMsg) + } + } } if o.SecretFile == "" { fmt.Printf("\nNo secret file was specified, no Secret created.\n\n") diff --git a/pkg/install/install.go b/pkg/install/install.go index 3f605d864e..167b9cef9b 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -175,6 +175,53 @@ func DeploymentIsReady(factory client.DynamicFactory, namespace string) (bool, e return isReady, err } +// DaemonSetIsReady will poll the kubernetes API server to ensure the restic daemonset is ready, i.e. that +// pods are scheduled and available on all of the the desired nodes. +func DaemonSetIsReady(factory client.DynamicFactory, namespace string) (bool, error) { + gvk := schema.FromAPIVersionAndKind(appsv1.SchemeGroupVersion.String(), "DaemonSet") + apiResource := metav1.APIResource{ + Name: "daemonsets", + Namespaced: true, + } + + c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, namespace) + if err != nil { + return false, errors.Wrapf(err, "Error creating client for daemonset polling") + } + + // declare this variable out of scope so we can return it + var isReady bool + var readyObservations int32 + + err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + unstructuredDaemonSet, err := c.Get("restic", metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, nil + } else if err != nil { + return false, errors.Wrap(err, "error waiting for daemonset to be ready") + } + + daemonSet := new(appsv1.DaemonSet) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredDaemonSet.Object, daemonSet); err != nil { + return false, errors.Wrap(err, "error converting daemonset from unstructured") + } + + if daemonSet.Status.NumberAvailable == daemonSet.Status.DesiredNumberScheduled { + readyObservations++ + } + + // Wait for 5 observations of the daemonset being "ready" to be consistent with our check for + // the deployment being ready. + if readyObservations > 4 { + isReady = true + return true, nil + } else { + return false, nil + } + }) + return isReady, err +} + // GroupResources groups resources based on whether the resources are CustomResourceDefinitions or other types of kubernetes objects // This is useful to wait for readiness before creating CRD objects func GroupResources(resources *unstructured.UnstructuredList) *ResourceGroup { From e464dba8baabeed0a06d80fd6da0b2eef5e7b662 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Mon, 9 Sep 2019 12:00:41 -0600 Subject: [PATCH 2/2] changelog Signed-off-by: Steve Kriss --- changelogs/unreleased/1859-skriss | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/1859-skriss diff --git a/changelogs/unreleased/1859-skriss b/changelogs/unreleased/1859-skriss new file mode 100644 index 0000000000..373869705a --- /dev/null +++ b/changelogs/unreleased/1859-skriss @@ -0,0 +1 @@ +velero install: if `--use-restic` and `--wait` are specified, wait up to a minute for restic daemonset to be ready