diff --git a/pkg/build/buildkit/build.go b/pkg/build/buildkit/build.go index 435351d..26bc122 100644 --- a/pkg/build/buildkit/build.go +++ b/pkg/build/buildkit/build.go @@ -38,7 +38,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/tsuru/deploy-agent/pkg/build" - "github.com/tsuru/deploy-agent/pkg/build/gc" + "github.com/tsuru/deploy-agent/pkg/build/downscaler" pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1" "github.com/tsuru/deploy-agent/pkg/util" ) @@ -80,7 +80,7 @@ func (b *BuildKit) WithKubernetesDiscovery(cs *kubernetes.Clientset, dcs dynamic b.kdopts = &opts if opts.ScaleStatefulset != "" { - gc.Run(cs, opts.PodSelector, opts.ScaleStatefulset) + downscaler.StartWorker(cs, opts.PodSelector, opts.ScaleStatefulset) } return b diff --git a/pkg/build/buildkit/k8s_autodiscovery.go b/pkg/build/buildkit/k8s_autodiscovery.go index 225ab86..9829ac2 100644 --- a/pkg/build/buildkit/k8s_autodiscovery.go +++ b/pkg/build/buildkit/k8s_autodiscovery.go @@ -16,6 +16,7 @@ import ( "github.com/moby/buildkit/client" pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1" + "github.com/tsuru/deploy-agent/pkg/build/metadata" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -29,15 +30,6 @@ import ( "k8s.io/klog" ) -const ( - DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time" - DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" - - TsuruAppNamespace = "tsuru" - TsuruAppNameLabelKey = "tsuru.io/app-name" - TsuruIsBuildLabelKey = "tsuru.io/is-build" -) - var ( noopFunc = func() {} @@ -85,7 +77,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts cfns = append(cfns, func() { klog.V(4).Infoln("Removing Tsuru app labels in the pod", pod.Name) - nerr := unsetTsuruAppLabelOnBuildKitPod(context.Background(), d.cs, pod.Name, pod.Namespace) + nerr := unsetTsuruAppLabelOnBuildKitPod(ctx, d.cs, pod.Name, pod.Namespace) if nerr != nil { klog.Errorf("failed to unset Tsuru app labels: %s", nerr) } @@ -189,7 +181,7 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert klog.V(4).Infof("Discovering the namespace where app %s is running on...", app) - tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{}) + tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(metadata.TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{}) if err != nil { return "", err } @@ -309,22 +301,22 @@ func setTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clientset patch, err := json.Marshal([]any{ map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)), "value": app, }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)), "value": strconv.FormatBool(true), }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)), "value": "", // set annotation value to empty rather than removing it, since it might not exist at first run }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildStartingLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildStartingLabelKey)), "value": strconv.FormatInt(time.Now().Unix(), 10), }, }) @@ -340,15 +332,15 @@ func unsetTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clients patch, err := json.Marshal([]any{ map[string]any{ "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)), }, map[string]any{ "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)), + "path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)), }, map[string]any{ "op": "replace", - "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)), + "path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)), "value": strconv.FormatInt(time.Now().Unix(), 10), }, }) diff --git a/pkg/build/gc/gc.go b/pkg/build/downscaler/downscaler.go similarity index 61% rename from pkg/build/gc/gc.go rename to pkg/build/downscaler/downscaler.go index 5fc72b7..b73d054 100644 --- a/pkg/build/gc/gc.go +++ b/pkg/build/downscaler/downscaler.go @@ -1,4 +1,8 @@ -package gc +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package downscaler import ( "context" @@ -6,28 +10,30 @@ import ( "strconv" "time" + "github.com/tsuru/deploy-agent/pkg/build/metadata" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/klog" ) -func Run(clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) { +func StartWorker(clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) { ctx := context.Background() go func() { for { - TickGC(ctx, clientset, podSelector, buildkitStefulset) + err := Run(ctx, clientset, podSelector, buildkitStefulset) + if err != nil { + klog.Errorf("failed to run downscaler tick: %s", err.Error()) + } time.Sleep(time.Minute * 5) } }() } -const DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" // TODO: move to other place - -func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) error { +func Run(ctx context.Context, clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) (err error) { defer func() { recoverErr := recover() - fmt.Println("print err", recoverErr) + err = fmt.Errorf("panic: %s", recoverErr) }() buildKitPods, err := clientset.CoreV1().Pods("*").List(ctx, v1.ListOptions{ @@ -41,12 +47,12 @@ func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, b maxEndtimeByNS := map[string]int64{} for _, pod := range buildKitPods.Items { - if pod.Annotations[DeployAgentLastBuildEndingTimeLabelKey] == "" { + if pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey] == "" { maxEndtimeByNS[pod.Namespace] = -1 // mark that namespace has least one pod of buildkit running continue } - maxUsage, err := strconv.ParseInt(pod.Annotations[DeployAgentLastBuildEndingTimeLabelKey], 10, 64) + maxUsage, err := strconv.ParseInt(pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey], 10, 64) if err != nil { klog.Errorf("failed to parseint: %s", err.Error()) continue @@ -81,6 +87,10 @@ func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, b continue } + if statefulset.Spec.Replicas != nil { + statefulset.Annotations[metadata.DeployAgentLastReplicasAnnotationKey] = fmt.Sprintf("%d", *statefulset.Spec.Replicas) + } + statefulset.Spec.Replicas = &zero _, err = clientset.AppsV1().StatefulSets(ns).Update(ctx, statefulset, v1.UpdateOptions{}) diff --git a/pkg/build/metadata/metadata.go b/pkg/build/metadata/metadata.go new file mode 100644 index 0000000..61e08e6 --- /dev/null +++ b/pkg/build/metadata/metadata.go @@ -0,0 +1,15 @@ +// Copyright 2024 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metadata + +const ( + DeployAgentLastReplicasAnnotationKey = "deploy-agent.tsuru.io/last-replicas" + DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time" + DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" + + TsuruAppNamespace = "tsuru" + TsuruAppNameLabelKey = "tsuru.io/app-name" + TsuruIsBuildLabelKey = "tsuru.io/is-build" +)