From 6c39c0d8addf1be5061f0da94de93ed5f3fbccfe Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Fri, 8 Sep 2023 10:08:36 -0500 Subject: [PATCH] Starting profiler for single binary with k8s metrics (#3981) * starting profile for single binary Signed-off-by: Daniel Rammer * updating datacatalog Signed-off-by: Daniel Rammer --------- Signed-off-by: Daniel Rammer --- cmd/single/start.go | 52 ++++++++++++++++++++++++++++----------------- go.mod | 4 ++-- go.sum | 4 ++-- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/cmd/single/start.go b/cmd/single/start.go index c394cdf257..6712e46892 100644 --- a/cmd/single/start.go +++ b/cmd/single/start.go @@ -2,39 +2,39 @@ package single import ( "context" - - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/flyteorg/flytepropeller/pkg/controller/executors" - "sigs.k8s.io/controller-runtime/pkg/manager" - - "github.com/flyteorg/flyteadmin/pkg/common" - "github.com/flyteorg/flyteadmin/plugins" - propellerEntrypoint "github.com/flyteorg/flytepropeller/pkg/controller" - propellerConfig "github.com/flyteorg/flytepropeller/pkg/controller/config" - "github.com/flyteorg/flytestdlib/contextutils" - "github.com/flyteorg/flytestdlib/promutils/labeled" - "github.com/flyteorg/flytestdlib/storage" - - "github.com/flyteorg/flytepropeller/pkg/signals" - webhookEntrypoint "github.com/flyteorg/flytepropeller/pkg/webhook" - webhookConfig "github.com/flyteorg/flytepropeller/pkg/webhook/config" + "net/http" datacatalogConfig "github.com/flyteorg/datacatalog/pkg/config" datacatalogRepo "github.com/flyteorg/datacatalog/pkg/repositories" datacatalog "github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice" "github.com/flyteorg/flyteadmin/pkg/clusterresource" + "github.com/flyteorg/flyteadmin/pkg/common" + "github.com/flyteorg/flyteadmin/plugins" "github.com/flyteorg/flyteadmin/pkg/runtime" adminServer "github.com/flyteorg/flyteadmin/pkg/server" adminScheduler "github.com/flyteorg/flyteadmin/scheduler" + propellerEntrypoint "github.com/flyteorg/flytepropeller/pkg/controller" + propellerConfig "github.com/flyteorg/flytepropeller/pkg/controller/config" + "github.com/flyteorg/flytepropeller/pkg/controller/executors" + "github.com/flyteorg/flytepropeller/pkg/signals" + webhookEntrypoint "github.com/flyteorg/flytepropeller/pkg/webhook" + webhookConfig "github.com/flyteorg/flytepropeller/pkg/webhook/config" + "github.com/flyteorg/flytestdlib/contextutils" "github.com/flyteorg/flytestdlib/logger" + "github.com/flyteorg/flytestdlib/profutils" "github.com/flyteorg/flytestdlib/promutils" + "github.com/flyteorg/flytestdlib/promutils/labeled" + "github.com/flyteorg/flytestdlib/storage" _ "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" _ "gorm.io/driver/postgres" // Required to import database driver. + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) const defaultNamespace = "all" @@ -147,6 +147,20 @@ func startPropeller(ctx context.Context, cfg Propeller) error { } if !cfg.DisableWebhook || !cfg.Disabled { + handlers := map[string]http.Handler{ + "/k8smetrics": promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ + ErrorHandling: promhttp.HTTPErrorOnError, + }), + } + + g.Go(func() error { + err := profutils.StartProfilingServerWithDefaultHandlers(childCtx, propellerCfg.ProfilerPort.Port, handlers) + if err != nil { + logger.Fatalf(childCtx, "Failed to Start profiling and metrics server. Error: %v", err) + } + return err + }) + g.Go(func() error { err := propellerEntrypoint.StartControllerManager(childCtx, mgr) if err != nil { diff --git a/go.mod b/go.mod index 713fbd4e48..a895e731b0 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/flyteorg/flyte go 1.19 require ( - github.com/flyteorg/datacatalog v1.0.51 + github.com/flyteorg/datacatalog v1.0.53 github.com/flyteorg/flyteadmin v1.1.118 github.com/flyteorg/flytepropeller v1.1.116 github.com/flyteorg/flytestdlib v1.0.22 github.com/golang/glog v1.1.0 + github.com/prometheus/client_golang v1.12.1 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 golang.org/x/sync v0.1.0 @@ -156,7 +157,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac // indirect - github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect diff --git a/go.sum b/go.sum index 9a84ed0316..bf25d02f3d 100644 --- a/go.sum +++ b/go.sum @@ -343,8 +343,8 @@ github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flyteorg/datacatalog v1.0.51 h1:p4NZ9Y6tby415nyl11H9PTjtzQmt4pn8g+wUTRaHie0= -github.com/flyteorg/datacatalog v1.0.51/go.mod h1:bTNS5x+IVNuwawEQMQCRsdnV2bt750nGq+dc6cmxF6w= +github.com/flyteorg/datacatalog v1.0.53 h1:h06iGGTFFPGzMCowW+HHmYSYjOiSbv73quxoEFY7H1s= +github.com/flyteorg/datacatalog v1.0.53/go.mod h1:bTNS5x+IVNuwawEQMQCRsdnV2bt750nGq+dc6cmxF6w= github.com/flyteorg/flyteadmin v1.1.118 h1:KBXC6DeDaTJze8aCu1CmFZmKMQOaXmLek43X09yg5sI= github.com/flyteorg/flyteadmin v1.1.118/go.mod h1:FzjmBGA3Wc1X/l3sjHD6Kgy2LEe16KHm1Mx0JPqWdAg= github.com/flyteorg/flyteidl v1.5.14 h1:+3ewipoOp82fPyIVgvvrMq1lorl5Kz3Lh6sh/a9+loI=