From 20ae7721397a79aa6768919f779a665ad87f4e97 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri <54188298+saimidu@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:51:55 -0700 Subject: [PATCH 1/7] feature: Add mlflow package (#441) --- .../v2.0.0/cpu.additional_packages_env.in | 2 ++ .../v2.0.0/gpu.additional_packages_env.in | 2 ++ test/test_artifacts/v2/mlflow.test.Dockerfile | 16 +++++++++++ .../v2/scripts/run_mlflow_tests.sh | 28 +++++++++++++++++++ test/test_dockerfile_based_harness.py | 2 ++ 5 files changed, 50 insertions(+) create mode 100644 build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in create mode 100644 build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in create mode 100644 test/test_artifacts/v2/mlflow.test.Dockerfile create mode 100644 test/test_artifacts/v2/scripts/run_mlflow_tests.sh diff --git a/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in new file mode 100644 index 00000000..98ed78d3 --- /dev/null +++ b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in @@ -0,0 +1,2 @@ +#Add any additional packages here +conda-forge::mlflow[version='>=2.13.0,<3.0'] diff --git a/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in new file mode 100644 index 00000000..e43d506d --- /dev/null +++ b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in @@ -0,0 +1,2 @@ +#Add any additional packages here +conda-forge::mlflow[version='>=2.13.2,<3.0'] diff --git a/test/test_artifacts/v2/mlflow.test.Dockerfile b/test/test_artifacts/v2/mlflow.test.Dockerfile new file mode 100644 index 00000000..ddd48772 --- /dev/null +++ b/test/test_artifacts/v2/mlflow.test.Dockerfile @@ -0,0 +1,16 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN python -c "import mlflow" + +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/mlflow/mlflow.git && \ + : + +WORKDIR "mlflow/" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_mlflow_tests.sh . +RUN chmod +x run_mlflow_tests.sh +# Run tests in run_matplotlib_tests.sh +CMD ["./run_mlflow_tests.sh"] diff --git a/test/test_artifacts/v2/scripts/run_mlflow_tests.sh b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh new file mode 100644 index 00000000..633f5411 --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Run examples for keras, pytorch, sklearn, tensorflow +cd examples + +# keras +cd keras/ +python train.py +cd - + +# pytorch +cd pytorch/ +python mnist_tensorboard_artifact.py +cd - + +# sklearn +for folder in "sklearn_autolog/" "sklearn_elasticnet_diabetes/linux/" "sklearn_elasticnet_wine/" "sklearn_logistic_regression/"; do + cd ${folder} + for file in *.py; do + python "$file" || exit $? + done + cd - +done + +# tensorflow +cd tensorflow/ +python train.py +cd - diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index 56253b2f..06773154 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -44,6 +44,7 @@ ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), ("serve.test.Dockerfile", ["serve-langchain"]), ("langchain-aws.test.Dockerfile", ["langchain-aws"]), + ("mlflow.test.Dockerfile", ["mlflow"]), ], ) def test_dockerfiles_for_cpu( @@ -83,6 +84,7 @@ def test_dockerfiles_for_cpu( ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), ("serve.test.Dockerfile", ["serve-langchain"]), ("langchain-aws.test.Dockerfile", ["langchain-aws"]), + ("mlflow.test.Dockerfile", ["mlflow"]), ], ) def test_dockerfiles_for_gpu( From 0fae45f4b48a080cb56835075d002e6dee74ea77 Mon Sep 17 00:00:00 2001 From: Tian Wang <133085652+aws-tianquaw@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:43:25 -0700 Subject: [PATCH 2/7] Create supported image versions table (#442) --- README.md | 2 +- supported_images.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 supported_images.md diff --git a/README.md b/README.md index 2e275ed9..55925b8c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## Amazon SageMaker Distribution Amazon SageMaker Distribution is a set of Docker images that include popular frameworks for machine learning, data -science and visualization. +science and visualization. For the list of supported SageMaker Distributions images, see [SageMaker Distributions Images](supported_images.md#supported-image-versions). These images come in two variants, CPU and GPU, and include deep learning frameworks like PyTorch, TensorFlow and Keras; popular Python packages like numpy, scikit-learn and pandas; and IDEs like Jupyter Lab. The distribution contains diff --git a/supported_images.md b/supported_images.md new file mode 100644 index 00000000..69094203 --- /dev/null +++ b/supported_images.md @@ -0,0 +1,25 @@ +## Supported Image Versions + +The tables below list the supported SageMaker Distribution image versions. SageMaker will only release functionality and security fixes for images versions present in the tables. When you are building images or pulling images from the ECR repository, it is highly recommended for you to choose the supported image versions from the tables below. + +### CPU Images + +| Image Major Version | Image Minor Version | ECR Image URI | +|---------------------|---------------------|-----------------------| +|1 |1.8 |public.ecr.aws/sagemaker/sagemaker-distribution:1.8-cpu | +|1 |1.7 |public.ecr.aws/sagemaker/sagemaker-distribution:1.7-cpu | +|1 |1.6 |public.ecr.aws/sagemaker/sagemaker-distribution:1.6-cpu | +|1 |1.5 |public.ecr.aws/sagemaker/sagemaker-distribution:1.5-cpu | +|1 |1.4 |public.ecr.aws/sagemaker/sagemaker-distribution:1.4-cpu | +|0 |0.12 |public.ecr.aws/sagemaker/sagemaker-distribution:0.12-cpu | + +### GPU Images + +| Image Major Version | Image Minor Version | ECR Image URI | +|---------------------|---------------------|-----------------------| +|1 |1.8 |public.ecr.aws/sagemaker/sagemaker-distribution:1.8-gpu | +|1 |1.7 |public.ecr.aws/sagemaker/sagemaker-distribution:1.7-gpu | +|1 |1.6 |public.ecr.aws/sagemaker/sagemaker-distribution:1.6-gpu | +|1 |1.5 |public.ecr.aws/sagemaker/sagemaker-distribution:1.5-gpu | +|1 |1.4 |public.ecr.aws/sagemaker/sagemaker-distribution:1.4-gpu | +|0 |0.12 |public.ecr.aws/sagemaker/sagemaker-distribution:0.12-gpu | \ No newline at end of file From 9695c95f65a651eb6c7b4976d5f44b706f9dbe00 Mon Sep 17 00:00:00 2001 From: Goga Koreli Date: Wed, 19 Jun 2024 16:34:03 -0700 Subject: [PATCH 3/7] feature: Make amazon q default language model (#443) --- template/v1/dirs/etc/jupyter/jupyter_server_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/template/v1/dirs/etc/jupyter/jupyter_server_config.py b/template/v1/dirs/etc/jupyter/jupyter_server_config.py index 0182cc23..63a4e1d8 100644 --- a/template/v1/dirs/etc/jupyter/jupyter_server_config.py +++ b/template/v1/dirs/etc/jupyter/jupyter_server_config.py @@ -24,5 +24,6 @@ module = __import__("amazon_sagemaker_sql_editor") module_location = os.path.dirname(module.__file__) c.LanguageServerManager.extra_node_roots = [f"{module_location}/sql-language-server"] + c.AiExtension.default_language_model = "amazon-q:q-developer" except: pass From c959f7c4f366d63481796198afd4a5ffb042dd1b Mon Sep 17 00:00:00 2001 From: Clayton Parnell <42805768+claytonparnell@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:59:18 -0400 Subject: [PATCH 4/7] Revert "feature: Make amazon q default language model (#443)" (#449) --- template/v1/dirs/etc/jupyter/jupyter_server_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/template/v1/dirs/etc/jupyter/jupyter_server_config.py b/template/v1/dirs/etc/jupyter/jupyter_server_config.py index 63a4e1d8..0182cc23 100644 --- a/template/v1/dirs/etc/jupyter/jupyter_server_config.py +++ b/template/v1/dirs/etc/jupyter/jupyter_server_config.py @@ -24,6 +24,5 @@ module = __import__("amazon_sagemaker_sql_editor") module_location = os.path.dirname(module.__file__) c.LanguageServerManager.extra_node_roots = [f"{module_location}/sql-language-server"] - c.AiExtension.default_language_model = "amazon-q:q-developer" except: pass From aae147a4a489a5b2d6ed2a211dbf3d1407debfda Mon Sep 17 00:00:00 2001 From: Ankit Kumar <148398256+kumarnzt@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:32:36 -0700 Subject: [PATCH 5/7] feature: Add amazon-sagemaker-jupyter-ai-q-developer and others (#447) --- .../v1/v1.9/v1.9.0/cpu.additional_packages_env.in | 2 ++ .../v1/v1.9/v1.9.0/gpu.additional_packages_env.in | 2 ++ .../amazon-q-developer-jupyterlab-ext.test.Dockerfile} | 2 +- .../amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile | 6 ++++++ 4 files changed, 11 insertions(+), 1 deletion(-) rename test/test_artifacts/{v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile => v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile} (60%) create mode 100644 test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile diff --git a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in index 37039a80..ed94e0be 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in @@ -1 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] +amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] diff --git a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in index 37039a80..ed94e0be 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in @@ -1 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] +amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] diff --git a/test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile b/test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile similarity index 60% rename from test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile rename to test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile index 1ef68a71..02599032 100644 --- a/test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile +++ b/test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile @@ -3,4 +3,4 @@ FROM $SAGEMAKER_DISTRIBUTION_IMAGE ARG MAMBA_DOCKERFILE_ACTIVATE=1 -CMD ["python", "-c", "import amazon_codewhisperer_jupyterlab_ext"] +CMD ["python", "-c", "import amazon_q_developer_jupyterlab_ext"] diff --git a/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile b/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile new file mode 100644 index 00000000..4c9ec976 --- /dev/null +++ b/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile @@ -0,0 +1,6 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +CMD ["python", "-c", "import amazon_sagemaker_jupyter_ai_q_developer"] From 525e4951e23a22b93c102c25f8aa91bacb2b3888 Mon Sep 17 00:00:00 2001 From: Sai Parthasarathy Miduthuri <54188298+saimidu@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:34:39 -0700 Subject: [PATCH 6/7] feature: Add sagemaker-mlflow package (#448) --- .../v2.0.0/cpu.additional_packages_env.in | 3 ++- .../v2.0.0/gpu.additional_packages_env.in | 1 + .../v2/sagemaker-mlflow.test.Dockerfile | 19 +++++++++++++++++++ .../v2/scripts/run_mlflow_tests.sh | 2 ++ .../v2/scripts/run_sagemaker_mlflow_tests.sh | 5 +++++ test/test_dockerfile_based_harness.py | 1 + 6 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile create mode 100644 test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh diff --git a/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in index 98ed78d3..7495d996 100644 --- a/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in @@ -1,2 +1,3 @@ #Add any additional packages here -conda-forge::mlflow[version='>=2.13.0,<3.0'] +conda-forge::mlflow[version='>=2.13.2,<3.0'] +conda-forge::sagemaker-mlflow[version='>=0.1.0,<1.0'] diff --git a/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in index e43d506d..7495d996 100644 --- a/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in +++ b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in @@ -1,2 +1,3 @@ #Add any additional packages here conda-forge::mlflow[version='>=2.13.2,<3.0'] +conda-forge::sagemaker-mlflow[version='>=0.1.0,<1.0'] diff --git a/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile b/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile new file mode 100644 index 00000000..c89064c7 --- /dev/null +++ b/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile @@ -0,0 +1,19 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN python -c "import sagemaker_mlflow" + +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/aws/sagemaker-mlflow.git && \ + : + +# For running sagemaker-mlflow tests, we need pytest +RUN micromamba install -y --freeze-installed -c conda-forge pytest + +WORKDIR "sagemaker-mlflow/" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_sagemaker_mlflow_tests.sh . +RUN chmod +x run_sagemaker_mlflow_tests.sh +# Run tests in run_matplotlib_tests.sh +CMD ["./run_sagemaker_mlflow_tests.sh"] diff --git a/test/test_artifacts/v2/scripts/run_mlflow_tests.sh b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh index 633f5411..069e2199 100644 --- a/test/test_artifacts/v2/scripts/run_mlflow_tests.sh +++ b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + # Run examples for keras, pytorch, sklearn, tensorflow cd examples diff --git a/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh b/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh new file mode 100644 index 00000000..bbcab0dd --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +pytest -s -rA -vv test/unit diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index 06773154..43e1ca99 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -85,6 +85,7 @@ def test_dockerfiles_for_cpu( ("serve.test.Dockerfile", ["serve-langchain"]), ("langchain-aws.test.Dockerfile", ["langchain-aws"]), ("mlflow.test.Dockerfile", ["mlflow"]), + ("sagemaker-mlflow.test.Dockerfile", ["sagemaker-mlflow"]), ], ) def test_dockerfiles_for_gpu( From 28f2c5d95e40c77e1da1d36f2778546cae50f0e1 Mon Sep 17 00:00:00 2001 From: Balaji Sankar <115105204+balajisankar15@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:31:08 -0700 Subject: [PATCH 7/7] fix: Specified channel name for new packages that are staged for v1.9.0 (#451) Co-authored-by: Balaji Sankar --- build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in | 4 ++-- build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in index ed94e0be..bd9b3d45 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in @@ -1,3 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] -amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] -amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] +conda-forge::amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +conda-forge::amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] diff --git a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in index ed94e0be..bd9b3d45 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in @@ -1,3 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] -amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] -amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] +conda-forge::amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +conda-forge::amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0']