Starting profiler for single binary with k8s metrics (flyteorg#3981)

* starting profile for single binary Signed-off-by: Daniel Rammer <[email protected]> * updating datacatalog Signed-off-by: Daniel Rammer <[email protected]> --------- Signed-off-by: Daniel Rammer <[email protected]>
MemVerge · Sep 8, 2023 · 6c39c0d · 6c39c0d
1 parent 9c91a6d
commit 6c39c0d
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 23 deletions.
diff --git a/cmd/single/start.go b/cmd/single/start.go
@@ -2,39 +2,39 @@ package single
 
 import (
 	"context"
-
-	"k8s.io/client-go/rest"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/flyteorg/flytepropeller/pkg/controller/executors"
-	"sigs.k8s.io/controller-runtime/pkg/manager"
-
-	"github.com/flyteorg/flyteadmin/pkg/common"
-	"github.com/flyteorg/flyteadmin/plugins"
-	propellerEntrypoint "github.com/flyteorg/flytepropeller/pkg/controller"
-	propellerConfig "github.com/flyteorg/flytepropeller/pkg/controller/config"
-	"github.com/flyteorg/flytestdlib/contextutils"
-	"github.com/flyteorg/flytestdlib/promutils/labeled"
-	"github.com/flyteorg/flytestdlib/storage"
-
-	"github.com/flyteorg/flytepropeller/pkg/signals"
-	webhookEntrypoint "github.com/flyteorg/flytepropeller/pkg/webhook"
-	webhookConfig "github.com/flyteorg/flytepropeller/pkg/webhook/config"
+	"net/http"
 
 	datacatalogConfig "github.com/flyteorg/datacatalog/pkg/config"
 	datacatalogRepo "github.com/flyteorg/datacatalog/pkg/repositories"
 	datacatalog "github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice"
 	"github.com/flyteorg/flyteadmin/pkg/clusterresource"
+	"github.com/flyteorg/flyteadmin/pkg/common"
+	"github.com/flyteorg/flyteadmin/plugins"
 	"github.com/flyteorg/flyteadmin/pkg/runtime"
 	adminServer "github.com/flyteorg/flyteadmin/pkg/server"
 	adminScheduler "github.com/flyteorg/flyteadmin/scheduler"
+	propellerEntrypoint "github.com/flyteorg/flytepropeller/pkg/controller"
+	propellerConfig "github.com/flyteorg/flytepropeller/pkg/controller/config"
+	"github.com/flyteorg/flytepropeller/pkg/controller/executors"
+	"github.com/flyteorg/flytepropeller/pkg/signals"
+	webhookEntrypoint "github.com/flyteorg/flytepropeller/pkg/webhook"
+	webhookConfig "github.com/flyteorg/flytepropeller/pkg/webhook/config"
+	"github.com/flyteorg/flytestdlib/contextutils"
 	"github.com/flyteorg/flytestdlib/logger"
+	"github.com/flyteorg/flytestdlib/profutils"
 	"github.com/flyteorg/flytestdlib/promutils"
+	"github.com/flyteorg/flytestdlib/promutils/labeled"
+	"github.com/flyteorg/flytestdlib/storage"
 	_ "github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 	_ "gorm.io/driver/postgres" // Required to import database driver.
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 const defaultNamespace = "all"
@@ -147,6 +147,20 @@ func startPropeller(ctx context.Context, cfg Propeller) error {
 	}
 
 	if !cfg.DisableWebhook || !cfg.Disabled {
+		handlers := map[string]http.Handler{
+			"/k8smetrics": promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{
+				ErrorHandling: promhttp.HTTPErrorOnError,
+			}),
+		}
+
+		g.Go(func() error {
+			err := profutils.StartProfilingServerWithDefaultHandlers(childCtx, propellerCfg.ProfilerPort.Port, handlers)
+			if err != nil {
+				logger.Fatalf(childCtx, "Failed to Start profiling and metrics server. Error: %v", err)
+			}
+			return err
+		})
+
 		g.Go(func() error {
 			err := propellerEntrypoint.StartControllerManager(childCtx, mgr)
 			if err != nil {

diff --git a/go.mod b/go.mod
@@ -3,11 +3,12 @@ module github.com/flyteorg/flyte
 go 1.19
 
 require (
-	github.com/flyteorg/datacatalog v1.0.51
+	github.com/flyteorg/datacatalog v1.0.53
 	github.com/flyteorg/flyteadmin v1.1.118
 	github.com/flyteorg/flytepropeller v1.1.116
 	github.com/flyteorg/flytestdlib v1.0.22
 	github.com/golang/glog v1.1.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/sync v0.1.0
@@ -156,7 +157,6 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20201205024021-ac21108117ac // indirect
-	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect

diff --git a/go.sum b/go.sum
@@ -343,8 +343,8 @@ github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flyteorg/datacatalog v1.0.51 h1:p4NZ9Y6tby415nyl11H9PTjtzQmt4pn8g+wUTRaHie0=
-github.com/flyteorg/datacatalog v1.0.51/go.mod h1:bTNS5x+IVNuwawEQMQCRsdnV2bt750nGq+dc6cmxF6w=
+github.com/flyteorg/datacatalog v1.0.53 h1:h06iGGTFFPGzMCowW+HHmYSYjOiSbv73quxoEFY7H1s=
+github.com/flyteorg/datacatalog v1.0.53/go.mod h1:bTNS5x+IVNuwawEQMQCRsdnV2bt750nGq+dc6cmxF6w=
 github.com/flyteorg/flyteadmin v1.1.118 h1:KBXC6DeDaTJze8aCu1CmFZmKMQOaXmLek43X09yg5sI=
 github.com/flyteorg/flyteadmin v1.1.118/go.mod h1:FzjmBGA3Wc1X/l3sjHD6Kgy2LEe16KHm1Mx0JPqWdAg=
 github.com/flyteorg/flyteidl v1.5.14 h1:+3ewipoOp82fPyIVgvvrMq1lorl5Kz3Lh6sh/a9+loI=