feat(backend): Enable logging for KFP components (#10288)

backend/Dockerfile

@@ -59,6 +59,7 @@ ARG COMMIT_SHA=unknown
 ENV COMMIT_SHA=${COMMIT_SHA}
 ARG TAG_NAME=unknown
 ENV TAG_NAME=${TAG_NAME}
+ENV LOG_LEVEL info 
 
 WORKDIR /bin
 
@@ -82,4 +83,4 @@ RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_confi
 EXPOSE 8888
 
 # Start the apiserver
-CMD /bin/apiserver --config=/config --sampleconfig=/config/sample_config.json -logtostderr=true
+CMD /bin/apiserver --config=/config --sampleconfig=/config/sample_config.json -logtostderr=true --logLevel=${LOG_LEVEL}

backend/Dockerfile.persistenceagent

@@ -49,5 +49,6 @@ ENV TTL_SECONDS_AFTER_WORKFLOW_FINISH 86400
 
 # NUM_WORKERS indicates now many worker goroutines
 ENV NUM_WORKERS 2
+ENV LOG_LEVEL info
 
-CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS}
+CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} --logLevel=${LOG_LEVEL}

backend/Dockerfile.scheduledworkflow

@@ -45,5 +45,6 @@ COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
 COPY --from=builder /tmp/NOTICES /third_party/NOTICES
 
 ENV NAMESPACE ""
+ENV LOG_LEVEL info
 
-CMD /bin/controller --logtostderr=true --namespace=${NAMESPACE}
+CMD /bin/controller --logtostderr=true --namespace=${NAMESPACE} --logLevel=${LOG_LEVEL}

backend/src/agent/persistence/main.go

@@ -30,6 +30,7 @@ import (
 
 var (
 	masterURL                     string
+	logLevel                      string
 	kubeconfig                    string
 	initializeTimeout             time.Duration
 	timeout                       time.Duration
@@ -47,6 +48,7 @@ var (
 )
 
 const (
+	logLevelFlagName                      = "logLevel"
 	kubeconfigFlagName                    = "kubeconfig"
 	masterFlagName                        = "master"
 	initializationTimeoutFlagName         = "initializeTimeout"
@@ -86,6 +88,16 @@ func main() {
 		log.Fatalf("Error building schedule clientset: %s", err.Error())
 	}
 
+	if logLevel == "" {
+		logLevel = "info"
+	}
+
+	level, err := log.ParseLevel(logLevel)
+	if err != nil {
+		log.Fatal("Invalid log level:", err)
+	}
+	log.SetLevel(level)
+
 	clientParam := util.ClientParameters{QPS: float64(cfg.QPS), Burst: cfg.Burst}
 	execInformer := util.NewExecutionInformerOrFatal(util.ArgoWorkflow, namespace, time.Second*30, clientParam)
 
@@ -131,6 +143,7 @@ func main() {
 func init() {
 	flag.StringVar(&kubeconfig, kubeconfigFlagName, "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	flag.StringVar(&masterURL, masterFlagName, "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
+	flag.StringVar(&logLevel, logLevelFlagName, "", "Defines the log level for the application.")
 	flag.DurationVar(&initializeTimeout, initializationTimeoutFlagName, 2*time.Minute, "Duration to wait for initialization of the ML pipeline API server.")
 	flag.DurationVar(&timeout, timeoutFlagName, 1*time.Minute, "Duration to wait for calls to complete.")
 	flag.StringVar(&mlPipelineAPIServerName, mlPipelineAPIServerNameFlagName, "ml-pipeline", "Name of the ML pipeline API server.")

backend/src/apiserver/main.go

@@ -41,12 +41,14 @@ import (
 	"github.com/kubeflow/pipelines/backend/src/apiserver/resource"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/server"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 )
 
 var (
+	logLevelFlag       = flag.String("logLevel", "", "Defines the log level for the application.")
 	rpcPortFlag        = flag.String("rpcPortFlag", ":8887", "RPC Port")
 	httpPortFlag       = flag.String("httpPortFlag", ":8888", "Http Proxy Port")
 	configPath         = flag.String("config", "", "Path to JSON file containing config")
@@ -77,6 +79,17 @@ func main() {
 		}
 	}
 
+	logLevel := *logLevelFlag
+	if logLevel == "" {
+		logLevel = "info"
+	}
+
+	level, err := log.ParseLevel(logLevel)
+	if err != nil {
+		log.Fatal("Invalid log level:", err)
+	}
+	log.SetLevel(level)
+
 	go startRpcServer(resourceManager)
 	startHttpProxy(resourceManager)
 

backend/src/crd/controller/scheduledworkflow/main.go

@@ -32,6 +32,7 @@ import (
 )
 
 var (
+	logLevel    string
 	masterURL   string
 	kubeconfig  string
 	namespace   string
@@ -53,6 +54,16 @@ func main() {
 	cfg.QPS = float32(clientQPS)
 	cfg.Burst = clientBurst
 
+	if logLevel == "" {
+		logLevel = "info"
+	}
+
+	level, err := log.ParseLevel(logLevel)
+	if err != nil {
+		log.Fatal("Invalid log level:", err)
+	}
+	log.SetLevel(level)
+
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		log.Fatalf("Error building kubernetes clientset: %s", err.Error())
@@ -102,6 +113,7 @@ func initEnv() {
 func init() {
 	initEnv()
 
+	flag.StringVar(&logLevel, "logLevel", "", "Defines the log level for the application.")
 	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
 	flag.StringVar(&namespace, "namespace", "", "The namespace name used for Kubernetes informers to obtain the listers.")

manifests/kustomize/base/installs/generic/pipeline-install-config.yaml

@@ -93,3 +93,4 @@ data:
   ## If this value doesn't include a unit abbreviation, the units will be assumed
   ## to be nanoseconds.
   ConMaxLifeTime: "120s"
+  LOG_LEVEL: "info"

manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml

@@ -17,6 +17,8 @@ spec:
     spec:
       containers:
       - env:
+        - name: LOG_LEVEL
+          value: "info"
         - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
           valueFrom:
             configMapKeyRef:

manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml

@@ -25,6 +25,8 @@ spec:
             value: "86400"
           - name: NUM_WORKERS
             value: "2"
+          - name: LOG_LEVEL
+            value: "info"
         image: gcr.io/ml-pipeline/persistenceagent:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent

manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml

@@ -20,6 +20,8 @@ spec:
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
         env:
+          - name: LOG_LEVEL
+            value: "info"
           - name: NAMESPACE
             valueFrom:
               fieldRef: