diff --git a/.github/workflows/test-example-notebooks.yaml b/.github/workflows/test-example-notebooks.yaml index 1b8f86120a..46d393600b 100644 --- a/.github/workflows/test-example-notebooks.yaml +++ b/.github/workflows/test-example-notebooks.yaml @@ -33,4 +33,5 @@ jobs: -i ./examples/pytorch/image-classification/create-pytorchjob.ipynb \ -o ./examples/pytorch/image-classification/create-pytorchjob-output.ipynb \ -p "namespace default" \ + -d jupyter==1.1.1 -d ipykernel==6.29.5 -d papermill==2.6.0 \ -k ./sdk/python diff --git a/examples/pytorch/image-classification/create-pytorchjob.ipynb b/examples/pytorch/image-classification/create-pytorchjob.ipynb index 49ee49b8b3..eb73fd1460 100644 --- a/examples/pytorch/image-classification/create-pytorchjob.ipynb +++ b/examples/pytorch/image-classification/create-pytorchjob.ipynb @@ -34,7 +34,7 @@ }, "outputs": [], "source": [ - "python_sdk='kubeflow-training'\n", + "training_python_sdk='kubeflow-training'\n", "namespace='kubeflow-user-example-com'" ] }, @@ -57,7 +57,7 @@ "source": [ "# TODO (andreyvelich): Change to release version when SDK with the new APIs is published.\n", "# Install Kubeflow Python SDK\n", - "!pip install {python_sdk}" + "!pip install {training_python_sdk}" ] }, { diff --git a/scripts/run-notebook.sh b/scripts/run-notebook.sh index f67deae6ca..79d0f635b2 100755 --- a/scripts/run-notebook.sh +++ b/scripts/run-notebook.sh @@ -20,7 +20,9 @@ NOTEBOOK_INPUT="" NOTEBOOK_OUTPUT="" PAPERMILL_PARAMS=() PAPERMILL_PARAM_YAML="" -PYTHON_SDK="git+https://github.com/kubeflow/training-operator.git#subdirectory=sdk/python" +TRAINING_PYTHON_SDK="git+https://github.com/kubeflow/training-operator.git#subdirectory=sdk/python" +PYTHON_DEPENDENCIES=() +PYTHON_REQUIREMENTS_FILE="" usage() { echo "Usage: $0 -i -o [-p \" \"...] [-y ]" @@ -30,18 +32,23 @@ usage() { echo " -p Papermill parameters (optional), pass param name and value pair (in quotes whitespace separated)" echo " -y Papermill parameters YAML file (optional)" echo " -k Kubeflow Training Operator Python SDK (optional)" + echo " -d Python dependencies args (optional)" + echo " -r Python dependencies requirements file (optional)" echo " -h Show this help message" + echo "NOTE: papermill, jupyter and ipykernel are required Python dependencies to run Notebooks. Dependencies can be passed through -d or -r" exit 1 } -while getopts "i:o:y:p:k:h:" opt; do +while getopts "i:o:y:p:k:r:d:h:" opt; do case "$opt" in - i) NOTEBOOK_INPUT="$OPTARG" ;; # -i for notebook input path - o) NOTEBOOK_OUTPUT="$OPTARG" ;; # -o for notebook output path - p) PAPERMILL_PARAMS+=("$OPTARG") ;; # -p for papermill parameters - y) PAPERMILL_PARAM_YAML="$OPTARG" ;; # -y for papermill parameter yaml path - k) PYTHON_SDK="$OPTARG" ;; # -k for training operator python sdk - h) usage ;; # -h for help (usage) + i) NOTEBOOK_INPUT="$OPTARG" ;; # -i for notebook input path + o) NOTEBOOK_OUTPUT="$OPTARG" ;; # -o for notebook output path + p) PAPERMILL_PARAMS+=("$OPTARG") ;; # -p for papermill parameters + y) PAPERMILL_PARAM_YAML="$OPTARG" ;; # -y for papermill parameter yaml path + k) TRAINING_PYTHON_SDK="$OPTARG" ;; # -k for training operator python sdk + d) PYTHON_DEPENDENCIES+=("$OPTARG") ;; # -d for passing python dependencies as args + r) PYTHON_REQUIREMENTS_FILE="$OPTARG" ;; # -k for passing python dependencies as requirements file + h) usage ;; # -h for help (usage) *) usage; exit 1 ;; esac done @@ -51,11 +58,27 @@ if [ -z "$NOTEBOOK_INPUT" ] || [ -z "$NOTEBOOK_OUTPUT" ]; then exit 1 fi -# Install Python dependencies to run Jupyter Notebooks with papermill -pip install jupyter ipykernel papermill==2.6.0 +# Check if we need to install dependencies +if [ ${#PYTHON_DEPENDENCIES[@]} -gt 0 ] || [ -n "$PYTHON_REQUIREMENTS_FILE" ]; then + pip_install_cmd="pip install" -papermill_cmd="papermill $NOTEBOOK_INPUT $NOTEBOOK_OUTPUT -p python_sdk $PYTHON_SDK" + for dep in "${PYTHON_DEPENDENCIES[@]}"; do + pip_install_cmd="$pip_install_cmd $dep" + done + if [ -n "$PYTHON_REQUIREMENTS_FILE" ]; then + pip_install_cmd="$pip_install_cmd -r $PYTHON_REQUIREMENTS_FILE" + fi + + echo "Installing Dependencies: $pip_install_cmd" + eval "$pip_install_cmd" + if [ $? -ne 0 ]; then + echo "Error: Failed to install dependencies." >&2 + exit 1 + fi +fi + +papermill_cmd="papermill $NOTEBOOK_INPUT $NOTEBOOK_OUTPUT -p training_python_sdk $TRAINING_PYTHON_SDK" # Add papermill parameters (param name and value) for param in "${PAPERMILL_PARAMS[@]}"; do papermill_cmd="$papermill_cmd -p $param" @@ -65,6 +88,12 @@ if [ -n "$PAPERMILL_PARAM_YAML" ]; then papermill_cmd="$papermill_cmd -y $PAPERMILL_PARAM_YAML" fi +# Check if papermill is installed +if ! command -v papermill &> /dev/null; then + echo "Error: papermill is not installed. Please install papermill to proceed. Python dependencies can be passed through -d or -r args" + exit 1 +fi + echo "Running command: $papermill_cmd" $papermill_cmd