feat: Use container based deployments only

pkg/redact/redactor.go

@@ -0,0 +1,7 @@
+package redact
+
+import "io"
+
+type Redactor interface {
+	Redact(input io.Reader, path string) io.Reader
+}

pkg/wandb/spec/release/cdk8s/operator.go → pkg/wandb/spec/operator/config.go

@@ -1,4 +1,4 @@
-package cdk8s
+package operator
 
 import (
 	"os"

pkg/wandb/spec/release/cdk8s/git.go

@@ -1,99 +1,7 @@
 package cdk8s
 
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"os/exec"
-	"strings"
+import "os/exec"
 
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-playground/validator/v10"
-	v1 "github.com/wandb/operator/api/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-const ClonePath = "/tmp/git"
-
-func GetCdk8sGitSpec(s interface{}) *Cdk8sGit {
-	spec := &Cdk8sGit{}
-	specBytes, _ := json.Marshal(s)
-
-	if err := json.Unmarshal(specBytes, spec); err != nil {
-		return nil
-	}
-
-	if err := spec.Validate(); err != nil {
-		return nil
-	}
-
-	return spec
-}
-
-type Cdk8sGit struct {
-	URL    string `validate:"required" json:"url"`
-	Branch string `json:"branch"`
-}
-
-func (c *Cdk8sGit) Validate() error {
-	return validator.New().Struct(c)
-}
-
-func (c *Cdk8sGit) Path() string {
-	path := strings.ReplaceAll(c.URL, "https://", "")
-	path = strings.ReplaceAll(path, "http://", "")
-	path = strings.ReplaceAll(path, ".git", "")
-	path = strings.ReplaceAll(path, "github.com", "")
-	return "/tmp/git/" + strings.Trim(path, "/")
-}
-
-func (c Cdk8sGit) Apply(
-	ctx context.Context,
-	client client.Client,
-	wandb *v1.WeightsAndBiases,
-	scheme *runtime.Scheme,
-	config map[string]interface{},
-) error {
-	os.MkdirAll(c.Path(), 0755)
-
-	rm := exec.Command("rm", "-rf", c.Path())
-	rm.Run()
-
-	fmt.Println(c.Path())
-
-	if err := clone(c.URL, "", c.Path()); err != nil {
-		return err
-	}
-
-	return Cdk8sLocal{Directory: c.Path()}.
-		Apply(ctx, client, wandb, scheme, config)
-}
-
-func clone(url string, branch string, to string) error {
-	g, err := git.PlainClone(to, false, &git.CloneOptions{
-		URL:      url,
-		Progress: os.Stdout,
-		Depth:    1,
-	})
-
-	if branch != "" {
-		hash, err := g.ResolveRevision(plumbing.Revision("HEAD"))
-		if err != nil {
-			return err
-		}
-
-		workTree, err := g.Worktree()
-		if err != nil {
-			return nil
-		}
-
-		return workTree.Checkout(&git.CheckoutOptions{
-			Hash: *hash,
-		})
-	}
-
-	return err
-}
+func GitCloneCmd(url string, folder string) *exec.Cmd {
+	return exec.Command("git", "clone", url, folder, "--depth", "1")
+}

pkg/wandb/spec/release/cdk8s/image.go

@@ -3,224 +3,72 @@ package cdk8s
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-	"os"
-	"time"
+	"strings"
 
 	"github.com/go-playground/validator/v10"
 	v1 "github.com/wandb/operator/api/v1"
-	batchv1 "k8s.io/api/batch/v1"
-	corev1 "k8s.io/api/core/v1"
-	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"github.com/wandb/operator/pkg/wandb/spec/release/k8s"
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
-func GetCdk8sJobSpec(s interface{}) *Cdk8sJobSpec {
-	spec := &Cdk8sJobSpec{}
-	specBytes, _ := json.Marshal(s)
-
-	if err := json.Unmarshal(specBytes, spec); err != nil {
-		return nil
-	}
-
-	if err := spec.Validate(); err != nil {
-		return nil
-	}
-
-	return spec
+type Git struct {
+	URL string `json:"url" validate:"required,url"`
 }
 
-type Cdk8sJobSpec struct {
-	Image string            `json:"image" validate:"required"`
-	Envs  map[string]string `json:"envs"`
+// A single application container that you want to run within a pod.
+type Cdk8sContainer struct {
+	// Container image name. More info:
+	// https://kubernetes.io/docs/concepts/containers/images
+	Image string `json:"image"`
+	// Map of environment variables to set in the container.
+	Envs map[string]string `json:"envs"`
+	// Run pnpm install before running generate and build
+	Git *Git `json:"git,omitempty"`
 }
 
-func (c *Cdk8sJobSpec) Validate() error {
+func (c Cdk8sContainer) Validate() error {
 	return validator.New().Struct(c)
 }
 
-func (s Cdk8sJobSpec) Apply(
+func (s Cdk8sContainer) Apply(
 	ctx context.Context,
 	c client.Client,
 	wandb *v1.WeightsAndBiases,
 	scheme *runtime.Scheme,
 	config map[string]interface{},
 ) error {
-	serviceAccount := os.Getenv("SERVICE_ACCOUNT_NAME")
-	if serviceAccount == "" {
-		serviceAccount = createAdminServiceAccount(ctx, c, wandb)
-	}
-
-	envs := []corev1.EnvVar{}
-	if s.Envs != nil {
-		for k, v := range s.Envs {
-			envs = append(envs, corev1.EnvVar{Name: k, Value: v})
-		}
-	}
-
-	tru := true
-	job := &batchv1.Job{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      wandb.GetName() + "-apply",
-			Namespace: wandb.GetNamespace(),
-		},
-		Spec: batchv1.JobSpec{
-			Template: corev1.PodTemplateSpec{
-				Spec: corev1.PodSpec{
-					ServiceAccountName:           serviceAccount,
-					AutomountServiceAccountToken: &tru,
-					InitContainers: []corev1.Container{
-						{
-							Name:    "generate",
-							Image:   s.Image,
-							Command: PnpmGenerateBuildCmd(config).Args,
-							Env:     envs,
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									MountPath: "/cdk8s/dist",
-									Name:      "manifests",
-								},
-							},
-						},
-					},
-					Containers: []corev1.Container{
-						{
-							Name:    "apply",
-							Image:   s.Image,
-							Command: KubectApplyCmd("/cdk8s/dist", wandb.GetNamespace()).Args,
-							Env:     envs,
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									MountPath: "/cdk8s/dist",
-									Name:      "manifests",
-								},
-							},
-						},
-					},
-					Volumes: []corev1.Volume{
-						{
-							Name: "manifests",
-							VolumeSource: corev1.VolumeSource{
-								EmptyDir: &corev1.EmptyDirVolumeSource{},
-							},
-						},
-					},
-					RestartPolicy: corev1.RestartPolicyNever,
-				},
-			},
-		},
-	}
-
-	if err := WaitForJobCompletion(ctx, wandb, c); err != nil {
-		fmt.Println(err)
-	}
-
-	deletePolicy := metav1.DeletePropagationBackground
-	deleteOptions := &client.DeleteOptions{PropagationPolicy: &deletePolicy}
-	if err := c.Delete(ctx, job, deleteOptions); err != nil {
-		fmt.Println(err)
-	}
-
-	if err := controllerutil.SetControllerReference(wandb, job, scheme); err != nil {
-		return err
-	}
-
-	if err := c.Create(ctx, job); err != nil {
-		return err
-	}
-
-	// Don't delete so we can debug better
-	if err := WaitForJobCompletion(ctx, wandb, c); err != nil {
-		fmt.Println(err)
-	}
-
-	return nil
-}
-
-func WaitForJobCompletion(ctx context.Context, wandb *v1.WeightsAndBiases, c client.Client) error {
-	job := &batchv1.Job{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      wandb.GetName() + "-apply",
-			Namespace: wandb.GetNamespace(),
-		},
-	}
-	for {
-		j := &batchv1.Job{}
-		err := c.Get(
-			ctx,
-			client.ObjectKey{
-				Namespace: job.GetNamespace(),
-				Name:      job.GetName(),
-			},
-			j,
-		)
-
-		if errors.IsNotFound(err) {
-			break
-		}
-
-		if j.Status.CompletionTime != nil || j.Status.Failed > 0 {
-			break
-		}
-
-		time.Sleep(10 * time.Second)
+	if s.Image == "" {
+		s.Image = "wandb/cdk8s:latest"
 	}
-	return nil
-}
 
-func createAdminServiceAccount(
-	ctx context.Context,
-	client client.Client,
-	wandb *v1.WeightsAndBiases,
-) string {
-	serviceAccount := "controller-admin"
-	sa := &corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      serviceAccount,
-			Namespace: wandb.GetNamespace(), // change this to your desired namespace
-		},
+	if s.Envs == nil {
+		s.Envs = map[string]string{}
 	}
-	client.Create(ctx, sa)
 
-	clusterRole := &rbacv1.ClusterRole{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "rbac.authorization.k8s.io/v1",
-			Kind:       "ClusterRole",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: serviceAccount + "-role",
-		},
-		Rules: []rbacv1.PolicyRule{
-			{
-				APIGroups: []string{"*"},
-				Verbs:     []string{"*"},
-				Resources: []string{"*"},
-			},
-		},
+	jsonConfig, _ := json.Marshal(config)
+	s.Envs["CONFIG"] = string(jsonConfig)
+
+	cmds := []string{}
+	if s.Git != nil {
+		cmds = append(cmds, "rm -rf /git")
+		cmds = append(cmds, strings.Join(GitCloneCmd(s.Git.URL, "/git").Args, " "))
+		cmds = append(cmds, "cd /git")
+		cmds = append(cmds, strings.Join(PnpmInstallCmd().Args, " "))
+		cmds = append(cmds, strings.Join(PnpmGenerateDevCmd(config).Args, " "))
+	} else {
+		cmds = append(cmds, strings.Join(PnpmGenerateBuildCmd(config).Args, " "))
 	}
-	client.Create(ctx, clusterRole)
-
-	roleBinding := &rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: serviceAccount + "-rolebinding",
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      sa.Name,
-				Namespace: sa.Namespace,
-			},
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: "rbac.authorization.k8s.io",
-			Kind:     "ClusterRole",
-			Name:     clusterRole.Name, // This is the clusterrole we created in the previous step
+	cmds = append(cmds, strings.Join(KubectApplyCmd("/cdk8s/dist", wandb.GetNamespace()).Args, " "))
+
+	container := k8s.ContainerSpec{
+		Image: s.Image,
+		Envs:  s.Envs,
+		Command: []string{
+			"/bin/bash",
+			"-c",
+			strings.Join(cmds, " && "),
 		},
 	}
-	client.Create(ctx, roleBinding)
-	return serviceAccount
+	return container.Apply(ctx, c, wandb, scheme, config)
 }