chore: reduce startup time and add skaffold (#5970)

* chore: add skaffold

* chore: add NATS retry

* chore: add skaffold

.dockerignore

@@ -1,11 +1,12 @@
 .git
 .gitignore
 .github
-LICENSE
 README.md
 Makefile
+build
 docs
 assets
 config
 choco
-test
+test
+skaffold.yaml

.gitignore

@@ -45,3 +45,5 @@ movie.mp4
 /dist
 .vscode
 
+
+build/_local/values.dev.yaml

build/_local/agent-server.Dockerfile

@@ -0,0 +1,46 @@
+###################################
+## Build
+###################################
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG GOMODCACHE="/root/.cache/go-build"
+ARG GOCACHE="/go/pkg"
+ARG SKAFFOLD_GO_GCFLAGS
+
+RUN apk --no-cache --update add ca-certificates && (rm -rf /var/cache/apk/* || 0)
+
+WORKDIR /app
+COPY . .
+RUN --mount=type=cache,target="$GOMODCACHE" \
+    --mount=type=cache,target="$GOCACHE" \
+    GOOS=$TARGETOS \
+    GOARCH=$TARGETARCH \
+    CGO_ENABLED=0 \
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/agent-server cmd/api-server/main.go
+
+###################################
+## Debug
+###################################
+FROM golang:1.23-alpine AS debug
+
+ENV GOTRACEBACK=all
+RUN go install github.com/go-delve/delve/cmd/[email protected]
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /app/build/_local/agent-server /testkube/
+
+ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/agent-server"]
+
+###################################
+## Distribution
+###################################
+FROM scratch AS dist
+
+COPY LICENSE /testkube/
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /app/build/_local/agent-server /testkube/
+
+EXPOSE 8080
+ENTRYPOINT ["/testkube/agent-server"]

build/_local/testworkflow-init.Dockerfile

@@ -0,0 +1,63 @@
+ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"
+
+###################################
+## Build testworkflow init
+###################################
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG GOMODCACHE="/root/.cache/go-build"
+ARG GOCACHE="/go/pkg"
+ARG SKAFFOLD_GO_GCFLAGS
+
+WORKDIR /app
+COPY . .
+RUN --mount=type=cache,target="$GOMODCACHE" \
+    --mount=type=cache,target="$GOCACHE" \
+    GOOS=$TARGETOS \
+    GOARCH=$TARGETARCH \
+    CGO_ENABLED=0 \
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go
+
+###################################
+## Build testworkflow toolkit
+###################################
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG GOMODCACHE="/root/.cache/go-build"
+ARG GOCACHE="/go/pkg"
+ARG SKAFFOLD_GO_GCFLAGS
+
+WORKDIR /app
+COPY . .
+RUN --mount=type=cache,target="$GOMODCACHE" \
+    --mount=type=cache,target="$GOCACHE" \
+    GOOS=$TARGETOS \
+    GOARCH=$TARGETARCH \
+    CGO_ENABLED=0 \
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go
+
+###################################
+## Debug
+###################################
+FROM golang:1.23-alpine AS debug
+
+ENV GOTRACEBACK=all
+RUN go install github.com/go-delve/delve/cmd/[email protected]
+
+COPY --from=builder /app/build/_local/workflow-init /testkube/
+
+ENTRYPOINT ["/go/bin/dlv", "exec", "--headless", "--continue", "--accept-multiclient", "--listen=:56268", "--api-version=2", "/testkube/workflow-init"]
+
+###################################
+## Distribution
+###################################
+FROM ${BUSYBOX_IMAGE} AS dist
+RUN cp -rf /bin /.tktw-bin
+COPY --from=builder /app/build/_local/workflow-init /testkube/init
+USER 1001
+ENTRYPOINT ["/init"]
+

build/_local/testworkflow-toolkit.Dockerfile

@@ -0,0 +1,71 @@
+ARG BUSYBOX_IMAGE="busybox:1.36.1-musl"
+ARG ALPINE_IMAGE="alpine:3.20.0"
+FROM ${BUSYBOX_IMAGE} AS busybox
+
+###################################
+## Build testworkflow-init
+###################################
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-init
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG GOMODCACHE="/root/.cache/go-build"
+ARG GOCACHE="/go/pkg"
+ARG SKAFFOLD_GO_GCFLAGS
+
+WORKDIR /app
+COPY . .
+RUN --mount=type=cache,target="$GOMODCACHE" \
+    --mount=type=cache,target="$GOCACHE" \
+    GOOS=$TARGETOS \
+    GOARCH=$TARGETARCH \
+    CGO_ENABLED=0 \
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-init cmd/testworkflow-init/main.go
+
+###################################
+## Build testworkflow-toolkit
+###################################
+FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder-toolkit
+
+ARG TARGETOS
+ARG TARGETARCH
+ARG GOMODCACHE="/root/.cache/go-build"
+ARG GOCACHE="/go/pkg"
+ARG SKAFFOLD_GO_GCFLAGS
+
+RUN go install github.com/go-delve/delve/cmd/[email protected]
+
+WORKDIR /app
+COPY . .
+RUN --mount=type=cache,target="$GOMODCACHE" \
+    --mount=type=cache,target="$GOCACHE" \
+    GOOS=$TARGETOS \
+    GOARCH=$TARGETARCH \
+    CGO_ENABLED=0 \
+    go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o build/_local/workflow-toolkit cmd/testworkflow-toolkit/main.go
+
+###################################
+## Debug
+###################################
+FROM ${ALPINE_IMAGE} AS debug
+RUN apk --no-cache add ca-certificates libssl3 git openssh-client
+ENV GOTRACEBACK=all
+COPY --from=builder-toolkit /go/bin/dlv /
+COPY --from=busybox /bin /.tktw-bin
+COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
+COPY --from=builder-init /app/build/_local/workflow-init /init
+RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
+USER 1001
+ENTRYPOINT ["/dlv", "exec", "--headless", "--accept-multiclient", "--listen=:56300", "--api-version=2", "/toolkit"]
+
+###################################
+## Distribution
+###################################
+FROM ${ALPINE_IMAGE} AS dist
+RUN apk --no-cache add ca-certificates libssl3 git openssh-client
+COPY --from=busybox /bin /.tktw-bin
+COPY --from=builder-toolkit /app/build/_local/workflow-toolkit /toolkit
+COPY --from=builder-init /app/build/_local/workflow-init /init
+RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
+USER 1001
+ENTRYPOINT ["/toolkit"]

build/_local/values.dev.tpl.yaml

@@ -0,0 +1,12 @@
+testkube-dashboard:
+  enabled: false
+mongodb:
+  enabled: false
+testkube-api:
+  cloud:
+    url: testkube-enterprise-api:8089
+    tls:
+      enabled: false
+    existingSecret:
+      name: testkube-default-agent-token
+      key: agent-token

docker-bake.hcl

@@ -0,0 +1,46 @@
+variable "GOCACHE" {
+  default = "/go/pkg"
+}
+variable "GOMODCACHE" {
+  default = "/root/.cache/go-build"
+}
+
+group "default" {
+  targets = ["agent-server", "testworkflow-init", "testworkflow-toolkit"]
+}
+
+target "agent-server-meta" {}
+target "agent-server" {
+  inherits = ["agent-server-meta"]
+  context="."
+  dockerfile = "build/_local/agent-server.Dockerfile"
+  platforms = ["linux/arm64"]
+  args = {
+    GOCACHE = "${GOCACHE}"
+    GOMODCACHE = "${GOMODCACHE}"
+  }
+}
+
+target "testworkflow-init-meta" {}
+target "testworkflow-init" {
+  inherits = ["testworkflow-init-meta"]
+  context="."
+  dockerfile = "build/_local/testworkflow-init.Dockerfile"
+  platforms = ["linux/arm64"]
+  args = {
+    GOCACHE = "${GOCACHE}"
+    GOMODCACHE = "${GOMODCACHE}"
+  }
+}
+
+target "testworkflow-toolkit-meta" {}
+target "testworkflow-toolkit" {
+  inherits = ["testworkflow-toolkit-meta"]
+  context="."
+  dockerfile = "build/_local/testworkflow-toolkit.Dockerfile"
+  platforms = ["linux/arm64"]
+  args = {
+    GOCACHE = "${GOCACHE}"
+    GOMODCACHE = "${GOMODCACHE}"
+  }
+}

go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/avast/retry-go/v4 v4.6.0 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect

go.sum

@@ -87,6 +87,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
+github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
+github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

pkg/event/bus/nats.go

@@ -6,15 +6,19 @@ import (
 	"sync"
 	"time"
 
+	"github.com/avast/retry-go/v4"
 	"github.com/nats-io/nats.go"
 
 	"github.com/kubeshop/testkube/pkg/api/v1/testkube"
 	"github.com/kubeshop/testkube/pkg/event/kind/common"
 	"github.com/kubeshop/testkube/pkg/log"
+	"github.com/kubeshop/testkube/pkg/utils"
 )
 
 var (
 	_ Bus = (*NATSBus)(nil)
+
+	NATS_RETRY_ATTEMPTS uint = 20
 )
 
 const (
@@ -80,7 +84,15 @@ func NewNATSEncodedConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.
 func NewNATSConnection(cfg ConnectionConfig, opts ...nats.Option) (*nats.Conn, error) {
 	opts = append(opts, optsFromConfig(cfg)...)
 
-	nc, err := nats.Connect(cfg.NatsURI, opts...)
+	log.DefaultLogger.Infoln("Connecting to NATS")
+	nc, err := retry.DoWithData(
+		func() (*nats.Conn, error) {
+			return nats.Connect(cfg.NatsURI, opts...)
+		},
+		retry.DelayType(retry.FixedDelay),
+		retry.Delay(utils.DefaultRetryDelay),
+		retry.Attempts(NATS_RETRY_ATTEMPTS),
+	)
 	if err != nil {
 		log.DefaultLogger.Fatalw("error connecting to nats", "error", err)
 		return nil, err

pkg/utils/consts.go

@@ -1,6 +1,9 @@
 package utils
 
+import "time"
+
 const (
 	// DefaultDockerRegistry is the default registry used when no registry is specified in the image name.
 	DefaultDockerRegistry = "https://index.docker.io/v1/"
+	DefaultRetryDelay     = time.Second * 3
 )

skaffold.yaml

@@ -0,0 +1,58 @@
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: testkube-agent
+build:
+  local:
+    concurrency: 1 # for testing, should be possible to run all concurrently.
+  artifacts:
+    - image: docker.io/testkube-agent-server
+      context: .
+      custom:
+        buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set agent-server.tags="$IMAGE" --set agent-server.target="debug" agent-server
+        dependencies:
+          dockerfile:
+            path: build/_local/agent-server.Dockerfile
+    - image: docker.io/testworkflow-init
+      context: .
+      custom:
+        buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-init.tags="$IMAGE" testworkflow-init
+        dependencies:
+          dockerfile:
+            path: build/_local/testworkflow-init.Dockerfile
+    - image: docker.io/testworkflow-toolkit
+      context: .
+      custom:
+        buildCommand: GOCACHE="$(go env GOCACHE)" GOMODCACHE="$(go env GOMODCACHE)" docker buildx bake --set testworkflow-toolkit.tags="$IMAGE" --set testworkflow-toolkit.target="debug" testworkflow-toolkit
+        dependencies:
+          dockerfile:
+            path: build/_local/testworkflow-toolkit.Dockerfile
+deploy:
+  helm:
+    # see https://skaffold.dev/docs/renderers/helm/#skaffoldyaml-configuration
+    releases:
+      - name: testkube-agent
+        repo: https://kubeshop.github.io/helm-charts
+        remoteChart: testkube
+        # Alternative: Local chart - useful for when you are actively making changes to the chart.
+#        chartPath: /Users/you/path/to/helm-charts/charts/testkube
+        upgradeOnChange: true
+#        skipBuildDependencies: true # This implies that you need to build dependencies yourself when you make local chart changes!
+        namespace: tk-dev
+        wait: true
+        createNamespace: true
+        valuesFiles: ['build/_local/values.dev.yaml'] # IMPORTANT: You will have to copy the values.dev.tpl.yaml template to get started!
+        setValueTemplates:
+          testkube-api.image.registry: '{{.IMAGE_DOMAIN_docker_io_testkube_agent_server}}'
+          testkube-api.image.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testkube_agent_server}}'
+          testkube-api.image.tag:  '{{.IMAGE_TAG_docker_io_testkube_agent_server}}@{{.IMAGE_DIGEST_docker_io_testkube_agent_server}}'
+          testkube-api.imageTwInit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_init}}'
+          testkube-api.imageTwInit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_init}}'
+          testkube-api.imageTwInit.tag:  '{{.IMAGE_TAG_docker_io_testworkflow_init}}@{{.IMAGE_DIGEST_docker_io_testworkflow_init}}'
+          testkube-api.imageTwToolkit.registry: '{{.IMAGE_DOMAIN_docker_io_testworkflow_toolkit}}'
+          testkube-api.imageTwToolkit.repository: '{{.IMAGE_REPO_NO_DOMAIN_docker_io_testworkflow_toolkit}}'
+          testkube-api.imageTwToolkit.tag:  '{{.IMAGE_TAG_docker_io_testworkflow_toolkit}}@{{.IMAGE_DIGEST_docker_io_testworkflow_toolkit}}'
+    flags:
+      upgrade: ["--no-hooks"]
+  statusCheckDeadlineSeconds: 300
+  tolerateFailuresUntilDeadline: true