From af9348722cda49aac844bc81a20caa57943be423 Mon Sep 17 00:00:00 2001 From: ddalvi Date: Tue, 28 Nov 2023 08:07:08 -0500 Subject: [PATCH 1/2] Parameterize v2 launcher image --- backend/src/v2/compiler/argocompiler/argo.go | 2 +- backend/src/v2/compiler/argocompiler/container.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index d7c488972ae..e93800dc8b1 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -117,7 +117,7 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S templates: make(map[string]*wfapi.Template), // TODO(chensun): release process and update the images. driverImage: "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707", - launcherImage: "gcr.io/ml-pipeline/kfp-launcher@sha256:50151a8615c8d6907aa627902dce50a2619fd231f25d1e5c2a72737a2ea4001e", + launcherImage: GetLauncherImage(), job: job, spec: spec, executors: deploy.GetExecutors(), diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index 4b115da9ff3..fb84a9051f0 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -16,6 +16,7 @@ package argocompiler import ( wfapi "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "os" "github.com/kubeflow/pipelines/api/v2alpha1/go/pipelinespec" "github.com/kubeflow/pipelines/backend/src/v2/component" k8score "k8s.io/api/core/v1" @@ -23,8 +24,11 @@ import ( const ( volumeNameKFPLauncher = "kfp-launcher" + DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4" + LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" ) + func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error { err := c.saveComponentSpec(name, component) if err != nil { @@ -52,6 +56,14 @@ type containerDriverInputs struct { kubernetesConfig string // optional, used when Kubernetes config is not empty } +func GetLauncherImage() string { + launcherImage := os.Getenv(LauncherImageEnvVar) + if launcherImage == "" { + launcherImage = DefaultLauncherImage + } + return launcherImage +} + func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { dagTask := &wfapi.DAGTask{ Name: name, @@ -226,7 +238,7 @@ func (c *workflowCompiler) addContainerExecutorTemplate() string { InitContainers: []wfapi.UserContainer{{ Container: k8score.Container{ Name: "kfp-launcher", - Image: c.launcherImage, + Image: GetLauncherImage(), Command: []string{"launcher-v2", "--copy", component.KFPLauncherPath}, VolumeMounts: []k8score.VolumeMount{{ Name: volumeNameKFPLauncher, From af2fa351366029c58e74bf43355cc709a445eb52 Mon Sep 17 00:00:00 2001 From: ddalvi Date: Mon, 4 Dec 2023 10:16:19 -0500 Subject: [PATCH 2/2] Parameterize v2 driver image --- backend/src/v2/compiler/argocompiler/argo.go | 2 +- backend/src/v2/compiler/argocompiler/container.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/src/v2/compiler/argocompiler/argo.go b/backend/src/v2/compiler/argocompiler/argo.go index e93800dc8b1..a5cfed5faef 100644 --- a/backend/src/v2/compiler/argocompiler/argo.go +++ b/backend/src/v2/compiler/argocompiler/argo.go @@ -116,8 +116,8 @@ func Compile(jobArg *pipelinespec.PipelineJob, kubernetesSpecArg *pipelinespec.S wf: wf, templates: make(map[string]*wfapi.Template), // TODO(chensun): release process and update the images. - driverImage: "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707", launcherImage: GetLauncherImage(), + driverImage: GetDriverImage(), job: job, spec: spec, executors: deploy.GetExecutors(), diff --git a/backend/src/v2/compiler/argocompiler/container.go b/backend/src/v2/compiler/argocompiler/container.go index fb84a9051f0..03d85f18019 100644 --- a/backend/src/v2/compiler/argocompiler/container.go +++ b/backend/src/v2/compiler/argocompiler/container.go @@ -26,9 +26,10 @@ const ( volumeNameKFPLauncher = "kfp-launcher" DefaultLauncherImage = "gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4" LauncherImageEnvVar = "V2_LAUNCHER_IMAGE" + DefaultDriverImage = "gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707" + DriverImageEnvVar = "V2_DRIVER_IMAGE" ) - func (c *workflowCompiler) Container(name string, component *pipelinespec.ComponentSpec, container *pipelinespec.PipelineDeploymentConfig_PipelineContainerSpec) error { err := c.saveComponentSpec(name, component) if err != nil { @@ -64,6 +65,14 @@ func GetLauncherImage() string { return launcherImage } +func GetDriverImage() string { + driverImage := os.Getenv(DriverImageEnvVar) + if driverImage == "" { + driverImage = DefaultDriverImage + } + return driverImage +} + func (c *workflowCompiler) containerDriverTask(name string, inputs containerDriverInputs) (*wfapi.DAGTask, *containerDriverOutputs) { dagTask := &wfapi.DAGTask{ Name: name, @@ -123,7 +132,7 @@ func (c *workflowCompiler) addContainerDriverTemplate() string { }, }, Container: &k8score.Container{ - Image: c.driverImage, + Image: GetDriverImage(), Command: []string{"driver"}, Args: []string{ "--type", "CONTAINER",