Wait for existence of a serviceAccount in zarf namespace before continuing

src/internal/cluster/seed.go

@@ -8,123 +8,11 @@ import (
 	"fmt"
 
 	"github.com/defenseunicorns/zarf/src/config"
-	"github.com/defenseunicorns/zarf/src/pkg/k8s"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
-	"github.com/defenseunicorns/zarf/src/pkg/pki"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	"github.com/defenseunicorns/zarf/src/types"
 )
 
-// InitZarfState initializes the Zarf state with the given temporary directory and init configs.
-func (c *Cluster) InitZarfState(tempPath types.TempPaths, initOptions types.ZarfInitOptions) error {
-	message.Debugf("package.preSeedRegistry(%#v)", tempPath)
-
-	var (
-		clusterArch string
-		distro      string
-		err         error
-	)
-
-	spinner := message.NewProgressSpinner("Gathering cluster information")
-	defer spinner.Stop()
-
-	spinner.Updatef("Getting cluster architecture")
-	if clusterArch, err = c.Kube.GetArchitecture(); err != nil {
-		spinner.Errorf(err, "Unable to validate the cluster system architecture")
-	}
-
-	// Attempt to load an existing state prior to init
-	// NOTE: We are ignoring the error here because we don't really expect a state to exist yet
-	spinner.Updatef("Checking cluster for existing Zarf deployment")
-	state, _ := c.LoadZarfState()
-
-	// If the distro isn't populated in the state, assume this is a new cluster
-	if state.Distro == "" {
-		spinner.Updatef("New cluster, no prior Zarf deployments found")
-
-		// If the K3s component is being deployed, skip distro detection
-		if initOptions.ApplianceMode {
-			distro = k8s.DistroIsK3s
-			state.ZarfAppliance = true
-		} else {
-			// Otherwise, trying to detect the K8s distro type
-			distro, err = c.Kube.DetectDistro()
-			if err != nil {
-				// This is a basic failure right now but likely could be polished to provide user guidance to resolve
-				return fmt.Errorf("unable to connect to the cluster to verify the distro: %w", err)
-			}
-		}
-
-		if distro != k8s.DistroIsUnknown {
-			spinner.Updatef("Detected K8s distro %s", distro)
-		}
-
-		// Defaults
-		state.Distro = distro
-		state.Architecture = clusterArch
-		state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen)
-
-		// Setup zarf agent PKI
-		state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost)
-
-		namespaces, err := c.Kube.GetNamespaces()
-		if err != nil {
-			return fmt.Errorf("unable to get the Kubernetes namespaces: %w", err)
-		}
-		// Mark existing namespaces as ignored for the zarf agent to prevent mutating resources we don't own
-		for _, namespace := range namespaces.Items {
-			spinner.Updatef("Marking existing namespace %s as ignored by Zarf Agent", namespace.Name)
-			if namespace.Labels == nil {
-				// Ensure label map exists to avoid nil panic
-				namespace.Labels = make(map[string]string)
-			}
-			// This label will tell the Zarf Agent to ignore this namespace
-			namespace.Labels[agentLabel] = "ignore"
-			if _, err = c.Kube.UpdateNamespace(&namespace); err != nil {
-				// This is not a hard failure, but we should log it
-				message.Errorf(err, "Unable to mark the namespace %s as ignored by Zarf Agent", namespace.Name)
-			}
-		}
-
-		// Try to create the zarf namespace
-		spinner.Updatef("Creating the Zarf namespace")
-		if _, err := c.Kube.CreateNamespace(ZarfNamespace, nil); err != nil {
-			spinner.Fatalf(err, "Unable to create the zarf namespace")
-		}
-	}
-
-	if clusterArch != state.Architecture {
-		return fmt.Errorf("cluster architecture %s does not match the Zarf state architecture %s", clusterArch, state.Architecture)
-	}
-
-	switch state.Distro {
-	case k8s.DistroIsK3s, k8s.DistroIsK3d:
-		state.StorageClass = "local-path"
-
-	case k8s.DistroIsKind, k8s.DistroIsGKE:
-		state.StorageClass = "standard"
-
-	case k8s.DistroIsDockerDesktop:
-		state.StorageClass = "hostpath"
-	}
-
-	if initOptions.StorageClass != "" {
-		state.StorageClass = initOptions.StorageClass
-	}
-
-	state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer)
-	state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo)
-
-	spinner.Success()
-
-	// Save the state back to K8s
-	if err := c.SaveZarfState(state); err != nil {
-		return fmt.Errorf("unable to save the Zarf state: %w", err)
-	}
-
-	return nil
-}
-
 // PostSeedRegistry handles cleanup once the seed registry is up.
 func (c *Cluster) PostSeedRegistry(tempPath types.TempPaths) error {
 	message.Debugf("cluster.PostSeedRegistry(%#v)", tempPath)

src/internal/cluster/state.go

@@ -7,11 +7,15 @@ package cluster
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/types"
 
+	"github.com/defenseunicorns/zarf/src/pkg/k8s"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/defenseunicorns/zarf/src/pkg/pki"
+	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -24,11 +28,126 @@ const (
 	ZarfPackageInfoLabel = "package-deploy-info"
 )
 
+// InitZarfState initializes the Zarf state with the given temporary directory and init configs.
+func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error {
+	var (
+		clusterArch string
+		distro      string
+		err         error
+	)
+
+	spinner := message.NewProgressSpinner("Gathering cluster information")
+	defer spinner.Stop()
+
+	spinner.Updatef("Getting cluster architecture")
+	if clusterArch, err = c.Kube.GetArchitecture(); err != nil {
+		spinner.Errorf(err, "Unable to validate the cluster system architecture")
+	}
+
+	// Attempt to load an existing state prior to init.
+	// NOTE: We are ignoring the error here because we don't really expect a state to exist yet.
+	spinner.Updatef("Checking cluster for existing Zarf deployment")
+	state, _ := c.LoadZarfState()
+
+	// If the distro isn't populated in the state, assume this is a new cluster.
+	if state.Distro == "" {
+		spinner.Updatef("New cluster, no prior Zarf deployments found")
+
+		// If the K3s component is being deployed, skip distro detection.
+		if initOptions.ApplianceMode {
+			distro = k8s.DistroIsK3s
+			state.ZarfAppliance = true
+		} else {
+			// Otherwise, trying to detect the K8s distro type.
+			distro, err = c.Kube.DetectDistro()
+			if err != nil {
+				// This is a basic failure right now but likely could be polished to provide user guidance to resolve.
+				return fmt.Errorf("unable to connect to the cluster to verify the distro: %w", err)
+			}
+		}
+
+		if distro != k8s.DistroIsUnknown {
+			spinner.Updatef("Detected K8s distro %s", distro)
+		}
+
+		// Defaults
+		state.Distro = distro
+		state.Architecture = clusterArch
+		state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen)
+
+		// Setup zarf agent PKI
+		state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost)
+
+		namespaces, err := c.Kube.GetNamespaces()
+		if err != nil {
+			return fmt.Errorf("unable to get the Kubernetes namespaces: %w", err)
+		}
+		// Mark existing namespaces as ignored for the zarf agent to prevent mutating resources we don't own.
+		for _, namespace := range namespaces.Items {
+			spinner.Updatef("Marking existing namespace %s as ignored by Zarf Agent", namespace.Name)
+			if namespace.Labels == nil {
+				// Ensure label map exists to avoid nil panic
+				namespace.Labels = make(map[string]string)
+			}
+			// This label will tell the Zarf Agent to ignore this namespace.
+			namespace.Labels[agentLabel] = "ignore"
+			if _, err = c.Kube.UpdateNamespace(&namespace); err != nil {
+				// This is not a hard failure, but we should log it.
+				message.Errorf(err, "Unable to mark the namespace %s as ignored by Zarf Agent", namespace.Name)
+			}
+		}
+
+		// Try to create the zarf namespace.
+		spinner.Updatef("Creating the Zarf namespace")
+		if _, err := c.Kube.CreateNamespace(ZarfNamespace, nil); err != nil {
+			return fmt.Errorf("unable to create the zarf namespace: %w", err)
+		}
+
+		// Wait up to 2 minutes for the default service account to be created.
+		// Some clusters seem to take a while to create this, see https://github.com/kubernetes/kubernetes/issues/66689.
+		// The default SA is required for pods to start properly.
+		if _, err := c.Kube.WaitForServiceAccount(ZarfNamespace, "default", 2*time.Minute); err != nil {
+			return fmt.Errorf("unable get default Zarf service account: %w", err)
+		}
+	}
+
+	if clusterArch != state.Architecture {
+		return fmt.Errorf("cluster architecture %s does not match the Zarf state architecture %s", clusterArch, state.Architecture)
+	}
+
+	switch state.Distro {
+	case k8s.DistroIsK3s, k8s.DistroIsK3d:
+		state.StorageClass = "local-path"
+
+	case k8s.DistroIsKind, k8s.DistroIsGKE:
+		state.StorageClass = "standard"
+
+	case k8s.DistroIsDockerDesktop:
+		state.StorageClass = "hostpath"
+	}
+
+	if initOptions.StorageClass != "" {
+		state.StorageClass = initOptions.StorageClass
+	}
+
+	state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer)
+	state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo)
+
+	spinner.Success()
+
+	// Save the state back to K8s
+	if err := c.SaveZarfState(state); err != nil {
+		return fmt.Errorf("unable to save the Zarf state: %w", err)
+	}
+
+	return nil
+}
+
 // LoadZarfState returns the current zarf/zarf-state secret data or an empty ZarfState.
 func (c *Cluster) LoadZarfState() (types.ZarfState, error) {
 	message.Debug("k8s.LoadZarfState()")
 
-	// The empty state that we will try to fill
+	// The empty state that we will try to fill.
 	state := types.ZarfState{}
 
 	// Set up the API connection
@@ -49,17 +168,17 @@ func (c *Cluster) SaveZarfState(state types.ZarfState) error {
 	message.Debugf("k8s.SaveZarfState()")
 	message.Debug(message.JSONValue(state))
 
-	// Convert the data back to JSON
+	// Convert the data back to JSON.
 	data, err := json.Marshal(state)
 	if err != nil {
 		return fmt.Errorf("unable to json-encode the zarf state")
 	}
 
-	// Set up the data wrapper
+	// Set up the data wrapper.
 	dataWrapper := make(map[string][]byte)
 	dataWrapper[ZarfStateDataKey] = data
 
-	// The secret object
+	// The secret object.
 	secret := &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: corev1.SchemeGroupVersion.String(),
@@ -76,7 +195,7 @@ func (c *Cluster) SaveZarfState(state types.ZarfState) error {
 		Data: dataWrapper,
 	}
 
-	// Attempt to create or update the secret and return
+	// Attempt to create or update the secret and return.
 	if err := c.Kube.CreateOrUpdateSecret(secret); err != nil {
 		return fmt.Errorf("unable to create the zarf state secret")
 	}

src/pkg/k8s/sa.go

@@ -6,8 +6,11 @@ package k8s
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -28,8 +31,32 @@ func (k *K8s) GetServiceAccount(namespace, name string) (*corev1.ServiceAccount,
 	return k.Clientset.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metaOptions)
 }
 
-// SaveServiceAccount updates the given service account in the cluster.
-func (k *K8s) SaveServiceAccount(svcAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error) {
+// UpdateServiceAccount updates the given service account in the cluster.
+func (k *K8s) UpdateServiceAccount(svcAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error) {
 	metaOptions := metav1.UpdateOptions{}
 	return k.Clientset.CoreV1().ServiceAccounts(svcAccount.Namespace).Update(context.TODO(), svcAccount, metaOptions)
 }
+
+// WaitForServiceAccount waits for a service account to be created in the cluster.
+func (k *K8s) WaitForServiceAccount(ns, name string, timeout time.Duration) (*corev1.ServiceAccount, error) {
+	expired := time.After(timeout)
+
+	for {
+		select {
+		case <-expired:
+			return nil, fmt.Errorf("timed out waiting for service account %s/%s to exist", ns, name)
+
+		default:
+			sa, err := k.Clientset.CoreV1().ServiceAccounts(ns).Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					time.Sleep(1 * time.Second)
+					continue
+				}
+				return nil, fmt.Errorf("error getting service account %s/%s: %w", ns, name, err)
+			}
+
+			return sa, nil
+		}
+	}
+}

src/pkg/packager/deploy.go

@@ -134,7 +134,7 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts []
 		if err != nil {
 			return charts, fmt.Errorf("unable to connect to the Kubernetes cluster: %w", err)
 		}
-		p.cluster.InitZarfState(p.tmp, p.cfg.InitOpts)
+		p.cluster.InitZarfState(p.cfg.InitOpts)
 	}
 
 	if hasExternalRegistry && (isSeedRegistry || isInjector || isRegistry) {