Handling controller restarts during policy validation (#782)

* Handling controller restarts during policy validation

Fixes #753

* Updated retry interval for policy validation and improved error on timeout

client/policy_validator.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -9,6 +10,7 @@ import (
 	"regexp"
 	"time"
 
+	"github.com/skupperproject/skupper/api/types"
 	v1alpha12 "github.com/skupperproject/skupper/pkg/apis/skupper/v1alpha1"
 	"github.com/skupperproject/skupper/pkg/event"
 	"github.com/skupperproject/skupper/pkg/generated/client/clientset/versioned/typed/skupper/v1alpha1"
@@ -314,7 +316,8 @@ func (p *PolicyAPIClient) execGet(args ...string) (*PolicyAPIResult, error) {
 	ctx, cn := context.WithTimeout(context.Background(), time.Second*30)
 	defer cn()
 	notEnabledErr := fmt.Errorf("Skupper is not enabled in namespace '%s'", p.cli.Namespace)
-	err := utils.RetryWithContext(ctx, time.Millisecond*100, func() (bool, error) {
+	policyRetryInterval := time.Millisecond * 200
+	err := utils.RetryWithContext(ctx, policyRetryInterval, func() (bool, error) {
 		_, err := p.cli.exec([]string{"get", "policies", "-h"}, p.cli.GetNamespace())
 		if err != nil {
 			if _, err := getRootObject(p.cli); err != nil && errors.IsNotFound(err) {
@@ -348,9 +351,25 @@ func (p *PolicyAPIClient) execGet(args ...string) (*PolicyAPIResult, error) {
 	fullArgs := []string{"get", "policies"}
 	fullArgs = append(fullArgs, args...)
 	fullArgs = append(fullArgs, "-o", "json")
-	out, err := p.cli.exec(fullArgs, p.cli.GetNamespace())
+	var out *bytes.Buffer
+	var retryErr error
+	err = utils.RetryWithContext(ctx, policyRetryInterval, func() (bool, error) {
+		out, err = p.cli.exec(fullArgs, p.cli.GetNamespace())
+		if err != nil {
+			if _, err := getRootObject(p.cli); err != nil && errors.IsNotFound(err) {
+				return true, notEnabledErr
+			}
+			retryErr = err
+			return false, nil
+		}
+		retryErr = nil
+		return true, nil
+	})
 	if err != nil {
-		return nil, fmt.Errorf("Policy validation error: %v", err)
+		if os.IsTimeout(err) {
+			return nil, fmt.Errorf("Policy validation error: %s not ready - %v", types.ControllerDeploymentName, retryErr)
+		}
+		return nil, notEnabledErr
 	}
 	res := &PolicyAPIResult{}
 	err = json.Unmarshal(out.Bytes(), res)