From 642fac144fe471fd2c891b04ea8a633bb5259e0f Mon Sep 17 00:00:00 2001 From: Michal Szadkowski Date: Fri, 25 Oct 2024 12:47:01 +0200 Subject: [PATCH 1/3] Add MPIJob MultiCluster docs --- .../en/docs/tasks/run/multikueue/kubeflow.md | 7 +++++ .../en/docs/tasks/run/multikueue/mpijob.md | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 site/content/en/docs/tasks/run/multikueue/mpijob.md diff --git a/site/content/en/docs/tasks/run/multikueue/kubeflow.md b/site/content/en/docs/tasks/run/multikueue/kubeflow.md index 8396fb1c24..6d502c3523 100644 --- a/site/content/en/docs/tasks/run/multikueue/kubeflow.md +++ b/site/content/en/docs/tasks/run/multikueue/kubeflow.md @@ -31,3 +31,10 @@ See [Training Operator Installation](https://www.kubeflow.org/docs/components/tr ## MultiKueue integration Once the setup is complete you can test it by running one of the Kubeflow Jobs e.g. PyTorchJob [`sample-pytorchjob.yaml`](/docs/tasks/run/kubeflow/pytorchjobs/#sample-pytorchjob). + + +## Working alongside MPI Operator +In order for MPI-operator and Training-operator to work on the same cluster it is required that: +1. `kubeflow.org_mpijobs.yaml` entry is removed from `base/crds/kustomization.yaml` - https://github.com/kubeflow/training-operator/issues/1930 +2. Training Operator deployment is modified to enable all kubeflow jobs except for MPI - https://github.com/kubeflow/training-operator/issues/1777 + \ No newline at end of file diff --git a/site/content/en/docs/tasks/run/multikueue/mpijob.md b/site/content/en/docs/tasks/run/multikueue/mpijob.md new file mode 100644 index 0000000000..ab65d6a74d --- /dev/null +++ b/site/content/en/docs/tasks/run/multikueue/mpijob.md @@ -0,0 +1,27 @@ +--- +title: "Run MPI Jobs in Multi-Cluster" +linkTitle: "MPI" +weight: 3 +date: 2024-10-25 +description: > + Run a MultiKueue scheduled MPI Jobs. +--- + +## Before you begin + +Check the [MultiKueue installation guide](/docs/tasks/manage/setup_multikueue) on how to properly setup MultiKueue clusters. + +For the proper setup and use it is required using at least Kueue v0.9.0 and for MPI Operator at least v0.6.0. + +### Installation on the Clusters + +{{% alert title="Note" color="primary" %}} +Note: While both MPI Operator and Training Operator must be running on the same cluster, there are special steps that has to be applied to Training Operator deployment. +See [Working alongside MPI Operator](/docs/tasks/run/multikueue/kubeflow#working-alongside-mpi-operator) for more details. +{{% /alert %}} + +See [MPI Operator Installation](https://www.kubeflow.org/docs/components/training/user-guides/mpi/#installation) for installation and configuration details of MPI Operator. + +## MultiKueue integration + +Once the setup is complete you can test it by running a MPI Job [`sample-mpijob.yaml`](/docs/tasks/run/kubeflow/mpijobs/#sample-mpijob). From e7b0741b9a4c0d08ed8d32d69976659d338f66ea Mon Sep 17 00:00:00 2001 From: Michal Szadkowski Date: Fri, 25 Oct 2024 13:14:53 +0200 Subject: [PATCH 2/3] Include managedBy feature --- site/content/en/docs/tasks/run/multikueue/mpijob.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/site/content/en/docs/tasks/run/multikueue/mpijob.md b/site/content/en/docs/tasks/run/multikueue/mpijob.md index ab65d6a74d..91c021f7c8 100644 --- a/site/content/en/docs/tasks/run/multikueue/mpijob.md +++ b/site/content/en/docs/tasks/run/multikueue/mpijob.md @@ -25,3 +25,13 @@ See [MPI Operator Installation](https://www.kubeflow.org/docs/components/trainin ## MultiKueue integration Once the setup is complete you can test it by running a MPI Job [`sample-mpijob.yaml`](/docs/tasks/run/kubeflow/mpijobs/#sample-mpijob). + + +### ManagedBy + +The feature allows you to disable the MPI Operator and delegate reconciliation of that job to the Kueue controller. +In order to change the controller that reconciles the job to the Kueue you need to set a value of that field to `kueue.x-k8s.io/multikueue`. + +However the `spec.runPolicy.managedBy` field is defaulted to `kueue.x-k8s.io/multikueue` automatically if following conditions are met. + +The `kueue.x-k8s.io/queue-name` annotation of the mpijob job points to a Local Queue, whose corresponding Cluster Queue uses the Multi Kueue admission check. \ No newline at end of file From eba045709cbd8b89a7461401ccc0de715e62cef0 Mon Sep 17 00:00:00 2001 From: Michal Szadkowski Date: Tue, 29 Oct 2024 09:46:30 +0100 Subject: [PATCH 3/3] Update after code review --- .../en/docs/tasks/run/multikueue/mpijob.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/site/content/en/docs/tasks/run/multikueue/mpijob.md b/site/content/en/docs/tasks/run/multikueue/mpijob.md index 91c021f7c8..038ff08b3a 100644 --- a/site/content/en/docs/tasks/run/multikueue/mpijob.md +++ b/site/content/en/docs/tasks/run/multikueue/mpijob.md @@ -1,6 +1,6 @@ --- title: "Run MPI Jobs in Multi-Cluster" -linkTitle: "MPI" +linkTitle: "MPIJob" weight: 3 date: 2024-10-25 description: > @@ -26,12 +26,11 @@ See [MPI Operator Installation](https://www.kubeflow.org/docs/components/trainin Once the setup is complete you can test it by running a MPI Job [`sample-mpijob.yaml`](/docs/tasks/run/kubeflow/mpijobs/#sample-mpijob). +{{% alert title="Note" color="primary" %}} +Note: Kueue defaults the `spec.runPolicy.managedBy` field to `kueue.x-k8s.io/multikueue` on the management cluster for MPIJob. -### ManagedBy - -The feature allows you to disable the MPI Operator and delegate reconciliation of that job to the Kueue controller. -In order to change the controller that reconciles the job to the Kueue you need to set a value of that field to `kueue.x-k8s.io/multikueue`. - -However the `spec.runPolicy.managedBy` field is defaulted to `kueue.x-k8s.io/multikueue` automatically if following conditions are met. +This allows the MPI Operator to ignore the Jobs managed by MultiKueue on the management cluster, and in particular skip Pod creation. -The `kueue.x-k8s.io/queue-name` annotation of the mpijob job points to a Local Queue, whose corresponding Cluster Queue uses the Multi Kueue admission check. \ No newline at end of file +The pods are created and the actual computation will happen on the mirror copy of the Job on the selected worker cluster. +The mirror copy of the Job does not have the field set. +{{% /alert %}} \ No newline at end of file