Pass python dependencies as args and pin versions

.github/workflows/test-example-notebooks.yaml

@@ -33,4 +33,5 @@ jobs:
           -i ./examples/pytorch/image-classification/create-pytorchjob.ipynb \
           -o ./examples/pytorch/image-classification/create-pytorchjob-output.ipynb \
           -p "namespace default" \
+          -d jupyter==1.1.1 -d ipykernel==6.29.5 -d papermill==2.6.0 \
           -k ./sdk/python

examples/pytorch/image-classification/create-pytorchjob.ipynb

@@ -34,7 +34,7 @@
    },
    "outputs": [],
    "source": [
-    "python_sdk='kubeflow-training'\n",
+    "training_python_sdk='kubeflow-training'\n",
     "namespace='kubeflow-user-example-com'"
    ]
   },
@@ -57,7 +57,7 @@
    "source": [
     "# TODO (andreyvelich): Change to release version when SDK with the new APIs is published.\n",
     "# Install Kubeflow Python SDK\n",
-    "!pip install {python_sdk}"
+    "!pip install {training_python_sdk}"
    ]
   },
   {

scripts/run-notebook.sh

@@ -20,7 +20,9 @@ NOTEBOOK_INPUT=""
 NOTEBOOK_OUTPUT=""
 PAPERMILL_PARAMS=()
 PAPERMILL_PARAM_YAML=""
-PYTHON_SDK="git+https://github.com/kubeflow/training-operator.git#subdirectory=sdk/python"
+TRAINING_PYTHON_SDK="git+https://github.com/kubeflow/training-operator.git#subdirectory=sdk/python"
+PYTHON_DEPENDENCIES=()
+PYTHON_REQUIREMENTS_FILE=""
 
 usage() {
   echo "Usage: $0 -i <input_notebook> -o <output_notebook> [-p \"<param> <value>\"...] [-y <params.yaml>]"
@@ -30,18 +32,23 @@ usage() {
   echo "  -p  Papermill parameters (optional), pass param name and value pair (in quotes whitespace separated)"
   echo "  -y  Papermill parameters YAML file (optional)"
   echo "  -k  Kubeflow Training Operator Python SDK (optional)"
+  echo "  -d  Python dependencies args (optional)"
+  echo "  -r  Python dependencies requirements file (optional)"
   echo "  -h  Show this help message"
+  echo "NOTE: papermill, jupyter and ipykernel are required Python dependencies to run Notebooks. Dependencies can be passed through -d or -r"
   exit 1
 }
 
-while getopts "i:o:y:p:k:h:" opt; do
+while getopts "i:o:y:p:k:r:d:h:" opt; do
   case "$opt" in
-    i) NOTEBOOK_INPUT="$OPTARG" ;;          # -i for notebook input path
-    o) NOTEBOOK_OUTPUT="$OPTARG" ;;         # -o for notebook output path
-    p) PAPERMILL_PARAMS+=("$OPTARG") ;;     # -p for papermill parameters
-    y) PAPERMILL_PARAM_YAML="$OPTARG" ;;    # -y for papermill parameter yaml path
-    k) PYTHON_SDK="$OPTARG" ;;              # -k for training operator python sdk
-    h) usage ;;                             # -h for help (usage)
+    i) NOTEBOOK_INPUT="$OPTARG" ;;            # -i for notebook input path
+    o) NOTEBOOK_OUTPUT="$OPTARG" ;;           # -o for notebook output path
+    p) PAPERMILL_PARAMS+=("$OPTARG") ;;       # -p for papermill parameters
+    y) PAPERMILL_PARAM_YAML="$OPTARG" ;;      # -y for papermill parameter yaml path
+    k) TRAINING_PYTHON_SDK="$OPTARG" ;;       # -k for training operator python sdk
+    d) PYTHON_DEPENDENCIES+=("$OPTARG") ;;    # -d for passing python dependencies as args
+    r) PYTHON_REQUIREMENTS_FILE="$OPTARG" ;;  # -k for passing python dependencies as requirements file
+    h) usage ;;                               # -h for help (usage)
     *) usage; exit 1 ;;
   esac
 done
@@ -51,11 +58,27 @@ if [ -z "$NOTEBOOK_INPUT" ] || [ -z "$NOTEBOOK_OUTPUT" ]; then
   exit 1
 fi
 
-# Install Python dependencies to run Jupyter Notebooks with papermill
-pip install jupyter ipykernel papermill==2.6.0
+# Check if we need to install dependencies
+if [ ${#PYTHON_DEPENDENCIES[@]} -gt 0 ] || [ -n "$PYTHON_REQUIREMENTS_FILE" ]; then
+  pip_install_cmd="pip install"
 
-papermill_cmd="papermill $NOTEBOOK_INPUT $NOTEBOOK_OUTPUT -p python_sdk $PYTHON_SDK"
+  for dep in "${PYTHON_DEPENDENCIES[@]}"; do
+    pip_install_cmd="$pip_install_cmd $dep"
+  done
 
+  if [ -n "$PYTHON_REQUIREMENTS_FILE" ]; then
+    pip_install_cmd="$pip_install_cmd -r $PYTHON_REQUIREMENTS_FILE"
+  fi
+
+  echo "Installing Dependencies: $pip_install_cmd"
+  eval "$pip_install_cmd"
+  if [ $? -ne 0 ]; then
+    echo "Error: Failed to install dependencies." >&2
+    exit 1
+  fi
+fi
+
+papermill_cmd="papermill $NOTEBOOK_INPUT $NOTEBOOK_OUTPUT -p training_python_sdk $TRAINING_PYTHON_SDK"
 # Add papermill parameters (param name and value)
 for param in "${PAPERMILL_PARAMS[@]}"; do
   papermill_cmd="$papermill_cmd -p $param"
@@ -65,6 +88,12 @@ if [ -n "$PAPERMILL_PARAM_YAML" ]; then
   papermill_cmd="$papermill_cmd -y $PAPERMILL_PARAM_YAML"
 fi
 
+# Check if papermill is installed
+if ! command -v papermill &> /dev/null; then
+  echo "Error: papermill is not installed. Please install papermill to proceed. Python dependencies can be passed through -d or -r args"
+  exit 1
+fi
+
 echo "Running command: $papermill_cmd"
 $papermill_cmd