diff --git a/README.md b/README.md index 2e275ed9..55925b8c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## Amazon SageMaker Distribution Amazon SageMaker Distribution is a set of Docker images that include popular frameworks for machine learning, data -science and visualization. +science and visualization. For the list of supported SageMaker Distributions images, see [SageMaker Distributions Images](supported_images.md#supported-image-versions). These images come in two variants, CPU and GPU, and include deep learning frameworks like PyTorch, TensorFlow and Keras; popular Python packages like numpy, scikit-learn and pandas; and IDEs like Jupyter Lab. The distribution contains diff --git a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in index 37039a80..bd9b3d45 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/cpu.additional_packages_env.in @@ -1 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] +conda-forge::amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +conda-forge::amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] diff --git a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in index 37039a80..bd9b3d45 100644 --- a/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in +++ b/build_artifacts/v1/v1.9/v1.9.0/gpu.additional_packages_env.in @@ -1 +1,3 @@ conda-forge::langchain-aws[version='>=0.1.1,<0.2.0'] +conda-forge::amazon-sagemaker-jupyter-ai-q-developer[version='>=1.0.3,<2.0'] +conda-forge::amazon-q-developer-jupyterlab-ext[version='>=3.1.0,<4.0'] diff --git a/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in new file mode 100644 index 00000000..7495d996 --- /dev/null +++ b/build_artifacts/v2/v2.0/v2.0.0/cpu.additional_packages_env.in @@ -0,0 +1,3 @@ +#Add any additional packages here +conda-forge::mlflow[version='>=2.13.2,<3.0'] +conda-forge::sagemaker-mlflow[version='>=0.1.0,<1.0'] diff --git a/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in new file mode 100644 index 00000000..7495d996 --- /dev/null +++ b/build_artifacts/v2/v2.0/v2.0.0/gpu.additional_packages_env.in @@ -0,0 +1,3 @@ +#Add any additional packages here +conda-forge::mlflow[version='>=2.13.2,<3.0'] +conda-forge::sagemaker-mlflow[version='>=0.1.0,<1.0'] diff --git a/supported_images.md b/supported_images.md new file mode 100644 index 00000000..69094203 --- /dev/null +++ b/supported_images.md @@ -0,0 +1,25 @@ +## Supported Image Versions + +The tables below list the supported SageMaker Distribution image versions. SageMaker will only release functionality and security fixes for images versions present in the tables. When you are building images or pulling images from the ECR repository, it is highly recommended for you to choose the supported image versions from the tables below. + +### CPU Images + +| Image Major Version | Image Minor Version | ECR Image URI | +|---------------------|---------------------|-----------------------| +|1 |1.8 |public.ecr.aws/sagemaker/sagemaker-distribution:1.8-cpu | +|1 |1.7 |public.ecr.aws/sagemaker/sagemaker-distribution:1.7-cpu | +|1 |1.6 |public.ecr.aws/sagemaker/sagemaker-distribution:1.6-cpu | +|1 |1.5 |public.ecr.aws/sagemaker/sagemaker-distribution:1.5-cpu | +|1 |1.4 |public.ecr.aws/sagemaker/sagemaker-distribution:1.4-cpu | +|0 |0.12 |public.ecr.aws/sagemaker/sagemaker-distribution:0.12-cpu | + +### GPU Images + +| Image Major Version | Image Minor Version | ECR Image URI | +|---------------------|---------------------|-----------------------| +|1 |1.8 |public.ecr.aws/sagemaker/sagemaker-distribution:1.8-gpu | +|1 |1.7 |public.ecr.aws/sagemaker/sagemaker-distribution:1.7-gpu | +|1 |1.6 |public.ecr.aws/sagemaker/sagemaker-distribution:1.6-gpu | +|1 |1.5 |public.ecr.aws/sagemaker/sagemaker-distribution:1.5-gpu | +|1 |1.4 |public.ecr.aws/sagemaker/sagemaker-distribution:1.4-gpu | +|0 |0.12 |public.ecr.aws/sagemaker/sagemaker-distribution:0.12-gpu | \ No newline at end of file diff --git a/test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile b/test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile similarity index 60% rename from test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile rename to test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile index 1ef68a71..02599032 100644 --- a/test/test_artifacts/v1/amazon-codewhisperer-jupyterlab-ext.test.Dockerfile +++ b/test/test_artifacts/v2/amazon-q-developer-jupyterlab-ext.test.Dockerfile @@ -3,4 +3,4 @@ FROM $SAGEMAKER_DISTRIBUTION_IMAGE ARG MAMBA_DOCKERFILE_ACTIVATE=1 -CMD ["python", "-c", "import amazon_codewhisperer_jupyterlab_ext"] +CMD ["python", "-c", "import amazon_q_developer_jupyterlab_ext"] diff --git a/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile b/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile new file mode 100644 index 00000000..4c9ec976 --- /dev/null +++ b/test/test_artifacts/v2/amazon-sagemaker-jupyter-ai-q-developer.test.Dockerfile @@ -0,0 +1,6 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +CMD ["python", "-c", "import amazon_sagemaker_jupyter_ai_q_developer"] diff --git a/test/test_artifacts/v2/mlflow.test.Dockerfile b/test/test_artifacts/v2/mlflow.test.Dockerfile new file mode 100644 index 00000000..ddd48772 --- /dev/null +++ b/test/test_artifacts/v2/mlflow.test.Dockerfile @@ -0,0 +1,16 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN python -c "import mlflow" + +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/mlflow/mlflow.git && \ + : + +WORKDIR "mlflow/" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_mlflow_tests.sh . +RUN chmod +x run_mlflow_tests.sh +# Run tests in run_matplotlib_tests.sh +CMD ["./run_mlflow_tests.sh"] diff --git a/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile b/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile new file mode 100644 index 00000000..c89064c7 --- /dev/null +++ b/test/test_artifacts/v2/sagemaker-mlflow.test.Dockerfile @@ -0,0 +1,19 @@ +ARG SAGEMAKER_DISTRIBUTION_IMAGE +FROM $SAGEMAKER_DISTRIBUTION_IMAGE + +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +RUN python -c "import sagemaker_mlflow" + +RUN sudo apt-get update && sudo apt-get install -y git && \ + git clone --recursive https://github.com/aws/sagemaker-mlflow.git && \ + : + +# For running sagemaker-mlflow tests, we need pytest +RUN micromamba install -y --freeze-installed -c conda-forge pytest + +WORKDIR "sagemaker-mlflow/" +COPY --chown=$MAMBA_USER:$MAMBA_USER scripts/run_sagemaker_mlflow_tests.sh . +RUN chmod +x run_sagemaker_mlflow_tests.sh +# Run tests in run_matplotlib_tests.sh +CMD ["./run_sagemaker_mlflow_tests.sh"] diff --git a/test/test_artifacts/v2/scripts/run_mlflow_tests.sh b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh new file mode 100644 index 00000000..069e2199 --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_mlflow_tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +# Run examples for keras, pytorch, sklearn, tensorflow +cd examples + +# keras +cd keras/ +python train.py +cd - + +# pytorch +cd pytorch/ +python mnist_tensorboard_artifact.py +cd - + +# sklearn +for folder in "sklearn_autolog/" "sklearn_elasticnet_diabetes/linux/" "sklearn_elasticnet_wine/" "sklearn_logistic_regression/"; do + cd ${folder} + for file in *.py; do + python "$file" || exit $? + done + cd - +done + +# tensorflow +cd tensorflow/ +python train.py +cd - diff --git a/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh b/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh new file mode 100644 index 00000000..bbcab0dd --- /dev/null +++ b/test/test_artifacts/v2/scripts/run_sagemaker_mlflow_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +pytest -s -rA -vv test/unit diff --git a/test/test_dockerfile_based_harness.py b/test/test_dockerfile_based_harness.py index 56253b2f..43e1ca99 100644 --- a/test/test_dockerfile_based_harness.py +++ b/test/test_dockerfile_based_harness.py @@ -44,6 +44,7 @@ ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), ("serve.test.Dockerfile", ["serve-langchain"]), ("langchain-aws.test.Dockerfile", ["langchain-aws"]), + ("mlflow.test.Dockerfile", ["mlflow"]), ], ) def test_dockerfiles_for_cpu( @@ -83,6 +84,8 @@ def test_dockerfiles_for_cpu( ("amazon_sagemaker_sql_editor.test.Dockerfile", ["amazon_sagemaker_sql_editor"]), ("serve.test.Dockerfile", ["serve-langchain"]), ("langchain-aws.test.Dockerfile", ["langchain-aws"]), + ("mlflow.test.Dockerfile", ["mlflow"]), + ("sagemaker-mlflow.test.Dockerfile", ["sagemaker-mlflow"]), ], ) def test_dockerfiles_for_gpu(