Skip to content

Commit

Permalink
feat: add support for OSS version in devbox (#6012)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rangoo94
rangoo94 authored Nov 6, 2024
1 parent 7eb0098 commit fd4b475
Show file tree
Hide file tree
Showing 11 changed files with 400 additions and 91 deletions.
2 changes: 1 addition & 1 deletion cmd/api-server/services/controlplane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ func CreateControlPlane(ctx context.Context, cfg *config.Config, features featur
}
}
if cfg.LogsBucket != "" {
exists, err := storageClient.BucketExists(ctx, cfg.StorageBucket)
exists, err := storageClient.BucketExists(ctx, cfg.LogsBucket)
if err != nil {
log.DefaultLogger.Errorw("Failed to check if the storage bucket exists", "error", err)
} else if !exists {
Expand Down
8 changes: 5 additions & 3 deletions cmd/tcl/kubectl-testkube/devbox/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ This utility is used to help with development of the Agent features (like Test W
* For local development Testkube Enterprise (Skaffold), consider `testkube login --api-uri-override=http://localhost:8099 --agent-uri-override=http://testkube-enterprise-api.tk-dev.svc.local:8089 --auth-uri-override=http://localhost:5556 --custom-auth`
* It's worth to create alias for that in own `.bashrc` or `.bash_profile`
* It's worth to pass a devbox name, like `-n dawid`, so it's not using random name
* For OSS version - run with `--oss` parameter

The CLI will print a dashboard link for the selected environment.

Expand All @@ -44,12 +45,13 @@ Aliases:
devbox, dev

Flags:
--agent-image string base agent image (default "kubeshop/testkube-api-server:latest")
--init-image string base init image (default "kubeshop/testkube-tw-init:latest")
-n, --name string devbox name (default "1730107481990508000")
-s, --sync strings synchronise resources at paths
--toolkit-image string base toolkit image (default "kubeshop/testkube-tw-toolkit:latest")
-o, --open open dashboard in browser
-O, --oss run open source version
--agent-image string base agent image (default "kubeshop/testkube-api-server:latest")
--init-image string base init image (default "kubeshop/testkube-tw-init:latest")
--toolkit-image string base toolkit image (default "kubeshop/testkube-tw-toolkit:latest")
```

## Example
Expand Down
133 changes: 90 additions & 43 deletions cmd/tcl/kubectl-testkube/devbox/command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const (

func NewDevBoxCommand() *cobra.Command {
var (
oss bool
rawDevboxName string
open bool
baseAgentImage string
Expand Down Expand Up @@ -74,6 +75,8 @@ func NewDevBoxCommand() *cobra.Command {
ui.Fail(errors.New("testkube repository not found"))
}

var err error

// Connect to cluster
cluster, err := devutils.NewCluster()
if err != nil {
Expand All @@ -86,33 +89,39 @@ func NewDevBoxCommand() *cobra.Command {
pterm.Error.Printfln("Failed to load config file: %s", err.Error())
return
}
cfg.CloudContext.AgentUri = "https://agent-dev.testkube.dev"
cloud, err := devutils.NewCloud(cfg.CloudContext, cmd)
if err != nil {
pterm.Error.Printfln("Failed to connect to Cloud: %s", err.Error())
return
var cloud *devutils.CloudObject
if !oss {
cloud, err = devutils.NewCloud(cfg.CloudContext, cmd)
if err != nil {
pterm.Error.Printfln("Failed to connect to Cloud: %s", err.Error())
return
}
}

// Detect obsolete devbox environments
if obsolete := cloud.ListObsolete(); len(obsolete) > 0 {
count := 0
for _, env := range obsolete {
err := cloud.DeleteEnvironment(env.Id)
if err != nil {
fmt.Printf("Failed to delete obsolete devbox environment (%s): %s\n", env.Name, err.Error())
continue
if !oss {
if obsolete := cloud.ListObsolete(); len(obsolete) > 0 {
count := 0
for _, env := range obsolete {
err := cloud.DeleteEnvironment(env.Id)
if err != nil {
fmt.Printf("Failed to delete obsolete devbox environment (%s): %s\n", env.Name, err.Error())
continue
}
cluster.Namespace(env.Name).Destroy()
count++
}
cluster.Namespace(env.Name).Destroy()
count++
fmt.Printf("Deleted %d/%d obsolete devbox environments\n", count, len(obsolete))
}
fmt.Printf("Deleted %d/%d obsolete devbox environments\n", count, len(obsolete))
}

// Initialize bare cluster resources
namespace := cluster.Namespace(fmt.Sprintf("devbox-%s", rawDevboxName))
interceptorPod := namespace.Pod("devbox-interceptor")
agentPod := namespace.Pod("devbox-agent")
binaryStoragePod := namespace.Pod("devbox-binary")
mongoPod := namespace.Pod("devbox-mongodb")
minioPod := namespace.Pod("devbox-minio")

// Initialize binaries
interceptorBin := devutils.NewBinary(InterceptorMainPath, cluster.OperatingSystem(), cluster.Architecture())
Expand All @@ -132,6 +141,8 @@ func NewDevBoxCommand() *cobra.Command {
interceptor := devutils.NewInterceptor(interceptorPod, baseInitImage, baseToolkitImage, interceptorBin)
agent := devutils.NewAgent(agentPod, cloud, baseAgentImage, baseInitImage, baseToolkitImage)
binaryStorage := devutils.NewBinaryStorage(binaryStoragePod, binaryStorageBin)
mongo := devutils.NewMongo(mongoPod)
minio := devutils.NewMinio(minioPod)
var env *client.Environment

// Cleanup
Expand Down Expand Up @@ -170,10 +181,12 @@ func NewDevBoxCommand() *cobra.Command {
}

// Create environment in the Cloud
fmt.Println("Creating environment in Cloud...")
env, err = cloud.CreateEnvironment(namespace.Name())
if err != nil {
fail(errors.Wrap(err, "failed to create Cloud environment"))
if !oss {
fmt.Println("Creating environment in Cloud...")
env, err = cloud.CreateEnvironment(namespace.Name())
if err != nil {
fail(errors.Wrap(err, "failed to create Cloud environment"))
}
}

// Create namespace
Expand All @@ -182,6 +195,18 @@ func NewDevBoxCommand() *cobra.Command {
fail(errors.Wrap(err, "failed to create namespace"))
}

// Create resources accessor
var resources devutils.ResourcesClient
if oss {
resources = devutils.NewDirectResourcesClient(cluster.KubeClient(), namespace.Name())
} else {
client, err := cloud.Client(env.Id)
if err != nil {
fail(errors.Wrap(err, "failed to create cloud client"))
}
resources = client
}

g, _ := errgroup.WithContext(ctx)
binaryStorageReadiness := make(chan struct{})

Expand Down Expand Up @@ -234,6 +259,36 @@ func NewDevBoxCommand() *cobra.Command {
return nil
})

if oss {
// Deploying Minio
g.Go(func() error {
fmt.Println("[Minio] Deploying...")
if err = minio.Create(ctx); err != nil {
fail(errors.Wrap(err, "failed to create Minio service"))
}
fmt.Println("[Minio] Waiting for readiness...")
if err = minio.WaitForReady(ctx); err != nil {
fail(errors.Wrap(err, "failed to create Minio service"))
}
fmt.Println("[Minio] Ready")
return nil
})

// Deploying Mongo
g.Go(func() error {
fmt.Println("[Mongo] Deploying...")
if err = mongo.Create(ctx); err != nil {
fail(errors.Wrap(err, "failed to create Mongo service"))
}
fmt.Println("[Mongo] Waiting for readiness...")
if err = mongo.WaitForReady(ctx); err != nil {
fail(errors.Wrap(err, "failed to create Mongo service"))
}
fmt.Println("[Mongo] Ready")
return nil
})
}

// Deploying the Agent
g.Go(func() error {
fmt.Println("[Agent] Building...")
Expand Down Expand Up @@ -349,14 +404,14 @@ func NewDevBoxCommand() *cobra.Command {
}()

workflowLabel := func(name string) string {
if !termlink.SupportsHyperlinks() {
if !termlink.SupportsHyperlinks() || oss {
return name
}
return name + " " + termlink.ColorLink("(open)", cloud.DashboardUrl(env.Slug, fmt.Sprintf("dashboard/test-workflows/%s", name)), "magenta")
}

templateLabel := func(name string) string {
if !termlink.SupportsHyperlinks() {
if !termlink.SupportsHyperlinks() || oss {
return name
}
return name + " " + termlink.ColorLink("(open)", cloud.DashboardUrl(env.Slug, fmt.Sprintf("dashboard/test-workflow-templates/%s", name)), "magenta")
Expand All @@ -369,63 +424,51 @@ func NewDevBoxCommand() *cobra.Command {
parallel <- struct{}{}
switch update.Op {
case devutils.CRDSyncUpdateOpCreate:
client, err := cloud.Client(env.Id)
if err != nil {
fail(errors.Wrap(err, "failed to create cloud client"))
}
if update.Template != nil {
update.Template.Spec.Events = nil // ignore Cronjobs
_, err := client.CreateTestWorkflowTemplate(*testworkflows.MapTemplateKubeToAPI(update.Template))
_, err := resources.CreateTestWorkflowTemplate(*testworkflows.MapTemplateKubeToAPI(update.Template))
if err != nil {
fmt.Printf("CRD Sync: creating template: %s: error: %s\n", templateLabel(update.Template.Name), err.Error())
} else {
fmt.Println("CRD Sync: created template:", templateLabel(update.Template.Name))
}
} else {
update.Workflow.Spec.Events = nil // ignore Cronjobs
_, err := client.CreateTestWorkflow(*testworkflows.MapKubeToAPI(update.Workflow))
_, err := resources.CreateTestWorkflow(*testworkflows.MapKubeToAPI(update.Workflow))
if err != nil {
fmt.Printf("CRD Sync: creating workflow: %s: error: %s\n", workflowLabel(update.Workflow.Name), err.Error())
} else {
fmt.Println("CRD Sync: created workflow:", workflowLabel(update.Workflow.Name))
}
}
case devutils.CRDSyncUpdateOpUpdate:
client, err := cloud.Client(env.Id)
if err != nil {
fail(errors.Wrap(err, "failed to create cloud client"))
}
if update.Template != nil {
update.Template.Spec.Events = nil // ignore Cronjobs
_, err := client.UpdateTestWorkflowTemplate(*testworkflows.MapTemplateKubeToAPI(update.Template))
_, err := resources.UpdateTestWorkflowTemplate(*testworkflows.MapTemplateKubeToAPI(update.Template))
if err != nil {
fmt.Printf("CRD Sync: updating template: %s: error: %s\n", templateLabel(update.Template.Name), err.Error())
} else {
fmt.Println("CRD Sync: updated template:", templateLabel(update.Template.Name))
}
} else {
update.Workflow.Spec.Events = nil
_, err := client.UpdateTestWorkflow(*testworkflows.MapKubeToAPI(update.Workflow))
_, err := resources.UpdateTestWorkflow(*testworkflows.MapKubeToAPI(update.Workflow))
if err != nil {
fmt.Printf("CRD Sync: updating workflow: %s: error: %s\n", workflowLabel(update.Workflow.Name), err.Error())
} else {
fmt.Println("CRD Sync: updated workflow:", workflowLabel(update.Workflow.Name))
}
}
case devutils.CRDSyncUpdateOpDelete:
client, err := cloud.Client(env.Id)
if err != nil {
fail(errors.Wrap(err, "failed to create cloud client"))
}
if update.Template != nil {
err := client.DeleteTestWorkflowTemplate(update.Template.Name)
err := resources.DeleteTestWorkflowTemplate(update.Template.Name)
if err != nil {
fmt.Printf("CRD Sync: deleting template: %s: error: %s\n", templateLabel(update.Template.Name), err.Error())
} else {
fmt.Println("CRD Sync: deleted template:", templateLabel(update.Template.Name))
}
} else {
err := client.DeleteTestWorkflow(update.Workflow.Name)
err := resources.DeleteTestWorkflow(update.Workflow.Name)
if err != nil {
fmt.Printf("CRD Sync: deleting workflow: %s: error: %s\n", workflowLabel(update.Workflow.Name), err.Error())
} else {
Expand Down Expand Up @@ -571,10 +614,13 @@ func NewDevBoxCommand() *cobra.Command {
}

color.Green.Println("Development box is ready. Took", time.Since(startTs).Truncate(time.Millisecond))
if termlink.SupportsHyperlinks() {
fmt.Println("Dashboard:", termlink.Link(cloud.DashboardUrl(env.Slug, "dashboard/test-workflows"), cloud.DashboardUrl(env.Slug, "dashboard/test-workflows")))
} else {
fmt.Println("Dashboard:", cloud.DashboardUrl(env.Slug, "dashboard/test-workflows"))
fmt.Println("Namespace:", namespace.Name())
if !oss {
if termlink.SupportsHyperlinks() {
fmt.Println("Dashboard:", termlink.Link(cloud.DashboardUrl(env.Slug, "dashboard/test-workflows"), cloud.DashboardUrl(env.Slug, "dashboard/test-workflows")))
} else {
fmt.Println("Dashboard:", cloud.DashboardUrl(env.Slug, "dashboard/test-workflows"))
}
}
if open {
openurl.Run(cloud.DashboardUrl(env.Slug, "dashboard/test-workflows"))
Expand Down Expand Up @@ -609,6 +655,7 @@ func NewDevBoxCommand() *cobra.Command {
cmd.Flags().StringVarP(&rawDevboxName, "name", "n", fmt.Sprintf("%d", time.Now().UnixNano()), "devbox name")
cmd.Flags().StringSliceVarP(&syncResources, "sync", "s", nil, "synchronise resources at paths")
cmd.Flags().BoolVarP(&open, "open", "o", false, "open dashboard in browser")
cmd.Flags().BoolVarP(&oss, "oss", "O", false, "run open source version")
cmd.Flags().StringVar(&baseInitImage, "init-image", "kubeshop/testkube-tw-init:latest", "base init image")
cmd.Flags().StringVar(&baseToolkitImage, "toolkit-image", "kubeshop/testkube-tw-toolkit:latest", "base toolkit image")
cmd.Flags().StringVar(&baseAgentImage, "agent-image", "kubeshop/testkube-api-server:latest", "base agent image")
Expand Down
Loading

0 comments on commit fd4b475

Please sign in to comment.