Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release: v1.9.0 #432

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions build_artifacts/v1/v1.9/v1.9.0/CHANGELOG-cpu.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Change log: 1.9.0(cpu)

## Upgrades:

Package | Previous Version | Current Version
---|---|---
boto3|1.34.51|1.34.106
ipython|8.22.2|8.25.0
ipywidgets|8.1.2|8.1.3
aws-glue-sessions|1.0.5|1.0.6
jupyter-ai|2.14.1|2.16.0
sagemaker-python-sdk|2.219.0|2.221.1
uvicorn|0.29.0|0.30.1

## What's new:

Package | Version
---|---
langchain-aws|0.1.6
19 changes: 19 additions & 0 deletions build_artifacts/v1/v1.9/v1.9.0/CHANGELOG-gpu.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Change log: 1.9.0(gpu)

## Upgrades:

Package | Previous Version | Current Version
---|---|---
boto3|1.34.51|1.34.106
ipython|8.22.2|8.25.0
ipywidgets|8.1.2|8.1.3
aws-glue-sessions|1.0.5|1.0.6
jupyter-ai|2.14.1|2.16.0
sagemaker-python-sdk|2.219.0|2.221.1
uvicorn|0.29.0|0.30.1

## What's new:

Package | Version
---|---
langchain-aws|0.1.6
199 changes: 199 additions & 0 deletions build_artifacts/v1/v1.9/v1.9.0/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
ARG TAG_FOR_BASE_MICROMAMBA_IMAGE
FROM mambaorg/micromamba:$TAG_FOR_BASE_MICROMAMBA_IMAGE

ARG CUDA_MAJOR_MINOR_VERSION=''
ARG ENV_IN_FILENAME
ARG ARG_BASED_ENV_IN_FILENAME

ARG AMZN_BASE="/opt/amazon/sagemaker"
ARG DB_ROOT_DIR="/opt/db"
ARG DIRECTORY_TREE_STAGE_DIR="${AMZN_BASE}/dir-staging"

ARG NB_USER="sagemaker-user"
ARG NB_UID=1000
ARG NB_GID=100

# https://www.openssl.org/source/
ARG FIPS_VALIDATED_SSL=3.0.8

ENV SAGEMAKER_LOGGING_DIR="/var/log/sagemaker/"
ENV STUDIO_LOGGING_DIR="/var/log/studio/"
ENV EDITOR="nano"

USER root
RUN usermod "--login=${NB_USER}" "--home=/home/${NB_USER}" --move-home "-u ${NB_UID}" "${MAMBA_USER}" && \
groupmod "--new-name=${NB_USER}" --non-unique "-g ${NB_GID}" "${MAMBA_USER}" && \
# Update the expected value of MAMBA_USER for the
# _entrypoint.sh consistency check.
echo "${NB_USER}" > "/etc/arg_mamba_user" && \
:
ENV MAMBA_USER=$NB_USER
ENV USER=$NB_USER

RUN apt-get update && apt-get upgrade -y && \
apt-get install -y --no-install-recommends sudo gettext-base wget curl unzip git rsync build-essential openssh-client nano cron less mandoc && \
# We just install tzdata below but leave default time zone as UTC. This helps packages like Pandas to function correctly.
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata krb5-user libkrb5-dev libsasl2-dev libsasl2-modules && \
chmod g+w /etc/passwd && \
echo "ALL ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
touch /etc/krb5.conf.lock && chown ${NB_USER}:${MAMBA_USER} /etc/krb5.conf* && \
# Note that we do NOT run `rm -rf /var/lib/apt/lists/*` here. If we did, anyone building on top of our images will
# not be able to run any `apt-get install` commands and that would hamper customizability of the images.
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
sudo ./aws/install && \
rm -rf aws awscliv2.zip && \
:
RUN echo "source /usr/local/bin/_activate_current_env.sh" | tee --append /etc/profile

# CodeEditor - create server, user data dirs
RUN mkdir -p /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data \
&& chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/sagemaker-code-editor-server-data /opt/amazon/sagemaker/sagemaker-code-editor-user-data

# create dir to store user data files
RUN mkdir -p /opt/amazon/sagemaker/user-data \
&& chown $MAMBA_USER:$MAMBA_USER /opt/amazon/sagemaker/user-data


# Merge in OS directory tree contents.
RUN mkdir -p ${DIRECTORY_TREE_STAGE_DIR}
COPY dirs/ ${DIRECTORY_TREE_STAGE_DIR}/
RUN rsync -a ${DIRECTORY_TREE_STAGE_DIR}/ / && \
rm -rf ${DIRECTORY_TREE_STAGE_DIR}

# CodeEditor - download the extensions
RUN mkdir -p /etc/code-editor/extensions && \
while IFS= read -r url || [ -n "$url" ]; do \
echo "Downloading extension from ${url}..." && \
wget --no-check-certificate -P /etc/code-editor/extensions "${url}"; \
done < /etc/code-editor/extensions.txt

USER $MAMBA_USER
COPY --chown=$MAMBA_USER:$MAMBA_USER $ENV_IN_FILENAME *.in /tmp/

# Make sure that $ENV_IN_FILENAME has a newline at the end before the `tee` command runs. Otherwise, nasty things
# will happen.
RUN if [[ -z $ARG_BASED_ENV_IN_FILENAME ]] ; \
then echo 'No ARG_BASED_ENV_IN_FILENAME passed' ; \
else envsubst < /tmp/$ARG_BASED_ENV_IN_FILENAME | tee --append /tmp/$ENV_IN_FILENAME ; \
fi

ARG CONDA_OVERRIDE_CUDA=$CUDA_MAJOR_MINOR_VERSION
RUN micromamba install -y --name base --file /tmp/$ENV_IN_FILENAME && \
micromamba clean --all --yes --force-pkgs-dirs && \
rm -rf /tmp/*.in


ARG MAMBA_DOCKERFILE_ACTIVATE=1
RUN sudo ln -s $(which python3) /usr/bin/python

# Update npm version
RUN npm i -g npm

# Configure CodeEditor - Install extensions and set preferences
RUN \
extensionloc=/opt/amazon/sagemaker/sagemaker-code-editor-server-data/extensions && mkdir -p "${extensionloc}" \
# Loop through all vsix files in /etc/code-editor/extensions and install them
&& for ext in /etc/code-editor/extensions/*.vsix; do \
echo "Installing extension ${ext}..."; \
sagemaker-code-editor --install-extension "${ext}" --extensions-dir "${extensionloc}" --server-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-server-data --user-data-dir /opt/amazon/sagemaker/sagemaker-code-editor-user-data; \
done \
# Copy the settings
&& cp /etc/code-editor/code_editor_machine_settings.json /opt/amazon/sagemaker/sagemaker-code-editor-server-data/data/Machine/settings.json

# Install glue kernels, and move to shared directory
# Also patching base kernel so Studio background code doesn't start session silently
RUN install-glue-kernels && \
SITE_PACKAGES=$(pip show aws-glue-sessions | grep Location | awk '{print $2}') && \
jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_pyspark --user && \
jupyter-kernelspec install $SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_spark --user && \
mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_pyspark /opt/conda/share/jupyter/kernels && \
mv /home/sagemaker-user/.local/share/jupyter/kernels/glue_spark /opt/conda/share/jupyter/kernels && \
sed -i '/if not store_history and (/i\ if "sm_analytics_runtime_check" in code:\n return await self._complete_cell()\n' \
"$SITE_PACKAGES/aws_glue_interactive_sessions_kernel/glue_kernel_base/BaseKernel.py"


# Patch glue kernels to use kernel wrapper
COPY patch_glue_pyspark.json /opt/conda/share/jupyter/kernels/glue_pyspark/kernel.json
COPY patch_glue_spark.json /opt/conda/share/jupyter/kernels/glue_spark/kernel.json

# Configure RTC - disable jupyter_collaboration by default
RUN jupyter labextension disable @jupyter/collaboration-extension

USER root
RUN HOME_DIR="/home/${NB_USER}/licenses" \
&& mkdir -p ${HOME_DIR} \
&& curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
&& unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \
&& cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \
&& chmod +x /usr/local/bin/testOSSCompliance \
&& chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \
&& ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} python \
&& rm -rf ${HOME_DIR}/oss_compliance*

# Create logging directories for supervisor
RUN mkdir -p $SAGEMAKER_LOGGING_DIR && \
chmod a+rw $SAGEMAKER_LOGGING_DIR && \
mkdir -p ${STUDIO_LOGGING_DIR} && \
chown ${NB_USER}:${MAMBA_USER} ${STUDIO_LOGGING_DIR}

# Clean up CodeEditor artifacts
RUN rm -rf /etc/code-editor

# Create supervisord runtime directory
RUN mkdir -p /var/run/supervisord && \
chmod a+rw /var/run/supervisord

# Create root directory for DB
# Create logging directories for supervisor
RUN mkdir -p $DB_ROOT_DIR && \
chmod a+rw $DB_ROOT_DIR

USER $MAMBA_USER
ENV PATH="/opt/conda/bin:/opt/conda/condabin:$PATH"
WORKDIR "/home/${NB_USER}"

# Install FIPS Provider for OpenSSL, on top of existing OpenSSL installation
# v3.0.8 is latest FIPS validated provider, so this is the one we install
# But we need to run tests against the installed version.
# see https://github.com/openssl/openssl/blob/master/README-FIPS.md https://www.openssl.org/source/
RUN INSTALLED_SSL=$(micromamba list | grep openssl | tr -s ' ' | cut -d ' ' -f 3 | head -n 1) && \
# download source code for installed, and FIPS validated openssl versions
curl -L https://www.openssl.org/source/openssl-$FIPS_VALIDATED_SSL.tar.gz > openssl-$FIPS_VALIDATED_SSL.tar.gz && \
curl -L https://www.openssl.org/source/openssl-$INSTALLED_SSL.tar.gz > openssl-$INSTALLED_SSL.tar.gz && \
tar -xf openssl-$FIPS_VALIDATED_SSL.tar.gz && tar -xf openssl-$INSTALLED_SSL.tar.gz && cd openssl-$FIPS_VALIDATED_SSL && \
# Configure both versions to enable FIPS and build
./Configure enable-fips --prefix=/opt/conda --openssldir=/opt/conda/ssl && make && \
cd ../openssl-$INSTALLED_SSL && \
./Configure enable-fips --prefix=/opt/conda --openssldir=/opt/conda/ssl && make && \
# Copy validated provider to installed version for testing
cp ../openssl-$FIPS_VALIDATED_SSL/providers/fips.so providers/. && \
cp ../openssl-$FIPS_VALIDATED_SSL/providers/fipsmodule.cnf providers/. && \
make tests && cd ../openssl-$FIPS_VALIDATED_SSL && \
# After tests pass, install FIPS provider and remove source code
make install_fips && cd .. && rm -rf ./openssl-*
# Create new config file with fips-enabled. Then user can override OPENSSL_CONF to enable FIPS
# e.g. export OPENSSL_CONF=/opt/conda/ssl/openssl-fips.cnf
RUN cp /opt/conda/ssl/openssl.cnf /opt/conda/ssl/openssl-fips.cnf && \
sed -i "s:# .include fipsmodule.cnf:.include /opt/conda/ssl/fipsmodule.cnf:" /opt/conda/ssl/openssl-fips.cnf && \
sed -i 's:# fips = fips_sect:fips = fips_sect:' /opt/conda/ssl/openssl-fips.cnf
ENV OPENSSL_MODULES=/opt/conda/lib64/ossl-modules/

# Install Kerberos.
# Make sure no dependency is added/updated
RUN pip install "krb5>=0.5.1,<0.6" && \
pip show krb5 | grep Require | xargs -i sh -c '[ $(echo {} | cut -d: -f2 | wc -w) -eq 0 ] '

# https://stackoverflow.com/questions/122327
RUN SYSTEM_PYTHON_PATH=$(python3 -c "from __future__ import print_function;import sysconfig; print(sysconfig.get_paths().get('purelib'))") && \
# Remove SparkRKernel as it's not supported \
jupyter-kernelspec remove -f -y sparkrkernel && \
# Patch Sparkmagic lib to support Custom Certificates \
# https://github.com/jupyter-incubator/sparkmagic/pull/435/files \
cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/configuration.py ${SYSTEM_PYTHON_PATH}/sparkmagic/utils/ && \
cp -a ${SYSTEM_PYTHON_PATH}/sagemaker_studio_analytics_extension/patches/reliablehttpclient.py ${SYSTEM_PYTHON_PATH}/sparkmagic/livyclientlib/reliablehttpclient.py && \
sed -i 's= "python"= "/opt/conda/bin/python"=g' /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
sed -i 's="Spark"="SparkMagic Spark"=g' /opt/conda/share/jupyter/kernels/sparkkernel/kernel.json && \
sed -i 's="PySpark"="SparkMagic PySpark"=g' /opt/conda/share/jupyter/kernels/pysparkkernel/kernel.json

ENV SHELL=/bin/bash
58 changes: 58 additions & 0 deletions build_artifacts/v1/v1.9/v1.9.0/RELEASE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Release notes: 1.9.0

Package | gpu| cpu
---|---|---
python|3.10.14|3.10.14
numpy|1.26.4|1.26.4
jinja2|3.1.4|3.1.4
pandas|2.1.4|2.1.4
altair|5.3.0|5.3.0
boto3|1.34.106|1.34.106
ipython|8.25.0|8.25.0
jupyter-lsp|2.2.5|2.2.5
jupyterlab|4.1.6|4.1.6
amazon-codewhisperer-jupyterlab-ext|2.0.2|2.0.2
jupyter-scheduler|2.5.2|2.5.2
amazon-sagemaker-jupyter-scheduler|3.0.11|3.0.11
amazon-sagemaker-sql-magic|0.1.1|0.1.1
jupyterlab-lsp|5.0.3|5.0.3
amazon_sagemaker_sql_editor|0.1.7|0.1.7
scipy|1.11.4|1.11.4
scikit-learn|1.4.2|1.4.2
pip|23.3.2|23.3.2
torchvision|0.15.2|0.15.2
autogluon|0.8.3|0.8.3
ipywidgets|8.1.3|8.1.3
notebook|7.1.3|7.1.3
aws-glue-sessions|1.0.6|1.0.6
conda|23.11.0|23.11.0
fastapi|0.110.3|0.110.3
langchain|0.1.9|0.1.9
jupyter-ai|2.16.0|2.16.0
jupyter-collaboration|1.1.0|1.1.0
jupyter-dash|0.4.2|0.4.2
jupyter-server-proxy|4.1.2|4.1.2
jupyterlab-git|0.50.0|0.50.0
keras|2.15.0|2.15.0
langchain-aws|0.1.6|0.1.6
matplotlib|3.8.4|3.8.4
nodejs|18.20.2|18.20.2
py-xgboost-gpu|1.7.6|
thrift_sasl|0.4.3|0.4.3
pyhive|0.7.0|0.7.0
python-gssapi|1.8.3|1.8.3
python-lsp-server|1.11.0|1.11.0
pytorch-gpu|2.0.0|
sagemaker-code-editor|1.1.0|1.1.0
sagemaker-headless-execution-driver|0.0.12|0.0.12
sagemaker-jupyterlab-emr-extension|0.1.9|0.1.9
sagemaker-jupyterlab-extension|0.3.2|0.3.2
sagemaker-kernel-wrapper|0.0.2|0.0.2
sagemaker-python-sdk|2.221.1|2.221.1
sagemaker-studio-analytics-extension|0.0.21|0.0.21
sasl|0.3.1|0.3.1
supervisor|4.2.5|4.2.5
tensorflow|2.15.0|2.15.0
uvicorn|0.30.1|0.30.1
pytorch| |2.0.0
py-xgboost-cpu| |1.7.6
53 changes: 53 additions & 0 deletions build_artifacts/v1/v1.9/v1.9.0/cpu.env.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# This file is auto-generated.
conda-forge::langchain-aws
conda-forge::jupyter-collaboration[version='>=1.1.0,<2.0.0']
conda-forge::sagemaker-code-editor[version='>=1.1.0,<2.0.0']
conda-forge::amazon_sagemaker_sql_editor[version='>=0.1.7,<1.0.0']
conda-forge::amazon-sagemaker-sql-magic[version='>=0.1.1,<1.0.0']
conda-forge::langchain[version='>=0.1.9,<1.0.0']
conda-forge::fastapi[version='>=0.110.3,<1.0.0']
conda-forge::uvicorn[version='>=0.29.0,<1.0.0']
conda-forge::pytorch[version='>=2.0.0,<3.0.0']
conda-forge::tensorflow[version='>=2.15.0,<3.0.0']
conda-forge::python[version='>=3.10.14,<3.11.0']
conda-forge::pip[version='>=23.3.2,<24.0.0']
conda-forge::torchvision[version='>=0.15.2,<1.0.0']
conda-forge::numpy[version='>=1.26.4,<2.0.0']
conda-forge::pandas[version='>=2.1.4,<3.0.0']
conda-forge::scikit-learn[version='>=1.4.2,<2.0.0']
conda-forge::jinja2[version='>=3.1.4,<4.0.0']
conda-forge::matplotlib[version='>=3.8.4,<4.0.0']
conda-forge::sagemaker-headless-execution-driver[version='>=0.0.12,<1.0.0']
conda-forge::ipython[version='>=8.22.2,<9.0.0']
conda-forge::scipy[version='>=1.11.4,<2.0.0']
conda-forge::keras[version='>=2.15.0,<3.0.0']
conda-forge::py-xgboost-cpu[version='>=1.7.6,<2.0.0']
conda-forge::jupyterlab[version='>=4.1.6,<5.0.0']
conda-forge::ipywidgets[version='>=8.1.2,<9.0.0']
conda-forge::conda[version='>=23.11.0,<24.0.0']
conda-forge::boto3[version='>=1.34.51,<2.0.0']
conda-forge::sagemaker-python-sdk[version='>=2.219.0,<3.0.0']
conda-forge::supervisor[version='>=4.2.5,<5.0.0']
conda-forge::autogluon[version='>=0.8.3,<1.0.0']
conda-forge::aws-glue-sessions[version='>=1.0.5,<2.0.0']
conda-forge::sagemaker-kernel-wrapper[version='>=0.0.2,<1.0.0']
conda-forge::jupyter-ai[version='>=2.14.1,<3.0.0']
conda-forge::jupyter-scheduler[version='>=2.5.2,<3.0.0']
conda-forge::nodejs[version='>=18.20.2,<19.0.0']
conda-forge::jupyter-lsp[version='>=2.2.5,<3.0.0']
conda-forge::jupyterlab-lsp[version='>=5.0.3,<6.0.0']
conda-forge::python-lsp-server[version='>=1.11.0,<2.0.0']
conda-forge::notebook[version='>=7.1.3,<8.0.0']
conda-forge::altair[version='>=5.3.0,<6.0.0']
conda-forge::sagemaker-studio-analytics-extension[version='>=0.0.21,<1.0.0']
conda-forge::jupyter-dash[version='>=0.4.2,<1.0.0']
conda-forge::sagemaker-jupyterlab-extension[version='>=0.3.2,<1.0.0']
conda-forge::sagemaker-jupyterlab-emr-extension[version='>=0.1.9,<1.0.0']
conda-forge::amazon-sagemaker-jupyter-scheduler[version='>=3.0.11,<4.0.0']
conda-forge::jupyter-server-proxy[version='>=4.1.2,<5.0.0']
conda-forge::amazon-codewhisperer-jupyterlab-ext[version='>=2.0.2,<3.0.0']
conda-forge::jupyterlab-git[version='>=0.50.0,<1.0.0']
conda-forge::sasl[version='>=0.3.1,<1.0.0']
conda-forge::thrift_sasl[version='>=0.4.3,<1.0.0']
conda-forge::pyhive[version='>=0.7.0,<1.0.0']
conda-forge::python-gssapi[version='>=1.8.3,<2.0.0']
Loading