Skip to content

Commit

Permalink
Rename GC to Downscaler
Browse files Browse the repository at this point in the history
  • Loading branch information
@wpjunior
wpjunior committed May 6, 2024
1 parent 9919f01 commit 6037b77
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 29 deletions.
4 changes: 2 additions & 2 deletions pkg/build/buildkit/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
"k8s.io/client-go/kubernetes"

"github.com/tsuru/deploy-agent/pkg/build"
"github.com/tsuru/deploy-agent/pkg/build/gc"
"github.com/tsuru/deploy-agent/pkg/build/downscaler"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/util"
)
Expand Down Expand Up @@ -80,7 +80,7 @@ func (b *BuildKit) WithKubernetesDiscovery(cs *kubernetes.Clientset, dcs dynamic
b.kdopts = &opts

if opts.ScaleStatefulset != "" {
gc.Run(cs, opts.PodSelector, opts.ScaleStatefulset)
downscaler.StartWorker(cs, opts.PodSelector, opts.ScaleStatefulset)
}

return b
Expand Down
28 changes: 10 additions & 18 deletions pkg/build/buildkit/k8s_autodiscovery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/moby/buildkit/client"
pb "github.com/tsuru/deploy-agent/pkg/build/grpc_build_v1"
"github.com/tsuru/deploy-agent/pkg/build/metadata"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -29,15 +30,6 @@ import (
"k8s.io/klog"
)

const (
DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time"
DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time"

TsuruAppNamespace = "tsuru"
TsuruAppNameLabelKey = "tsuru.io/app-name"
TsuruIsBuildLabelKey = "tsuru.io/is-build"
)

var (
noopFunc = func() {}

Expand Down Expand Up @@ -85,7 +77,7 @@ func (d *k8sDiscoverer) discoverBuildKitClientFromApp(ctx context.Context, opts

cfns = append(cfns, func() {
klog.V(4).Infoln("Removing Tsuru app labels in the pod", pod.Name)
nerr := unsetTsuruAppLabelOnBuildKitPod(context.Background(), d.cs, pod.Name, pod.Namespace)
nerr := unsetTsuruAppLabelOnBuildKitPod(ctx, d.cs, pod.Name, pod.Namespace)
if nerr != nil {
klog.Errorf("failed to unset Tsuru app labels: %s", nerr)
}
Expand Down Expand Up @@ -189,7 +181,7 @@ func (d *k8sDiscoverer) buildkitPodNamespace(ctx context.Context, opts Kubernert

klog.V(4).Infof("Discovering the namespace where app %s is running on...", app)

tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{})
tsuruApp, err := d.dcs.Resource(tsuruAppGVR).Namespace(metadata.TsuruAppNamespace).Get(ctx, app, metav1.GetOptions{})
if err != nil {
return "", err
}
Expand Down Expand Up @@ -309,22 +301,22 @@ func setTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clientset
patch, err := json.Marshal([]any{
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)),
"value": app,
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)),
"value": strconv.FormatBool(true),
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)),
"value": "", // set annotation value to empty rather than removing it, since it might not exist at first run
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildStartingLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildStartingLabelKey)),
"value": strconv.FormatInt(time.Now().Unix(), 10),
},
})
Expand All @@ -340,15 +332,15 @@ func unsetTsuruAppLabelOnBuildKitPod(ctx context.Context, cs *kubernetes.Clients
patch, err := json.Marshal([]any{
map[string]any{
"op": "remove",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruAppNameLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruAppNameLabelKey)),
},
map[string]any{
"op": "remove",
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(TsuruIsBuildLabelKey)),
"path": fmt.Sprintf("/metadata/labels/%s", normalizeAppLabelForJSONPatch(metadata.TsuruIsBuildLabelKey)),
},
map[string]any{
"op": "replace",
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(DeployAgentLastBuildEndingTimeLabelKey)),
"path": fmt.Sprintf("/metadata/annotations/%s", normalizeAppLabelForJSONPatch(metadata.DeployAgentLastBuildEndingTimeLabelKey)),
"value": strconv.FormatInt(time.Now().Unix(), 10),
},
})
Expand Down
28 changes: 19 additions & 9 deletions pkg/build/gc/gc.go → pkg/build/downscaler/downscaler.go
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,39 @@
package gc
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package downscaler

import (
"context"
"fmt"
"strconv"
"time"

"github.com/tsuru/deploy-agent/pkg/build/metadata"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"
)

func Run(clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) {
func StartWorker(clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) {
ctx := context.Background()

go func() {
for {
TickGC(ctx, clientset, podSelector, buildkitStefulset)
err := Run(ctx, clientset, podSelector, buildkitStefulset)
if err != nil {
klog.Errorf("failed to run downscaler tick: %s", err.Error())
}
time.Sleep(time.Minute * 5)
}
}()
}

const DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time" // TODO: move to other place

func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) error {
func Run(ctx context.Context, clientset *kubernetes.Clientset, podSelector, buildkitStefulset string) (err error) {
defer func() {
recoverErr := recover()
fmt.Println("print err", recoverErr)
err = fmt.Errorf("panic: %s", recoverErr)
}()

buildKitPods, err := clientset.CoreV1().Pods("*").List(ctx, v1.ListOptions{
Expand All @@ -41,12 +47,12 @@ func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, b
maxEndtimeByNS := map[string]int64{}

for _, pod := range buildKitPods.Items {
if pod.Annotations[DeployAgentLastBuildEndingTimeLabelKey] == "" {
if pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey] == "" {
maxEndtimeByNS[pod.Namespace] = -1 // mark that namespace has least one pod of buildkit running
continue
}

maxUsage, err := strconv.ParseInt(pod.Annotations[DeployAgentLastBuildEndingTimeLabelKey], 10, 64)
maxUsage, err := strconv.ParseInt(pod.Annotations[metadata.DeployAgentLastBuildEndingTimeLabelKey], 10, 64)
if err != nil {
klog.Errorf("failed to parseint: %s", err.Error())
continue
Expand Down Expand Up @@ -81,6 +87,10 @@ func TickGC(ctx context.Context, clientset *kubernetes.Clientset, podSelector, b
continue
}

if statefulset.Spec.Replicas != nil {
statefulset.Annotations[metadata.DeployAgentLastReplicasAnnotationKey] = fmt.Sprintf("%d", *statefulset.Spec.Replicas)
}

statefulset.Spec.Replicas = &zero

_, err = clientset.AppsV1().StatefulSets(ns).Update(ctx, statefulset, v1.UpdateOptions{})
Expand Down
15 changes: 15 additions & 0 deletions pkg/build/metadata/metadata.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright 2024 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package metadata

const (
DeployAgentLastReplicasAnnotationKey = "deploy-agent.tsuru.io/last-replicas"
DeployAgentLastBuildStartingLabelKey = "deploy-agent.tsuru.io/last-build-starting-time"
DeployAgentLastBuildEndingTimeLabelKey = "deploy-agent.tsuru.io/last-build-ending-time"

TsuruAppNamespace = "tsuru"
TsuruAppNameLabelKey = "tsuru.io/app-name"
TsuruIsBuildLabelKey = "tsuru.io/is-build"
)

0 comments on commit 6037b77

Please sign in to comment.