Reactor/stable entrypoint

.github/workflows/e2e-test.yml

@@ -156,12 +156,6 @@ jobs:
         working-directory: ./scripts/e2e_test
         run: bash start_test.sh upload_pypi_to_nexus
 
-      - name: Build runtime image
-        working-directory: ./scripts/e2e_test
-        run: |
-          eval $(minikube docker-env)
-          bash start_test.sh build_runtime_image
-
       - name: Wait and check starwhale service
         working-directory: ./scripts/e2e_test
         run: bash start_test.sh check_controller_service

.github/workflows/release-base.yml

@@ -1,7 +1,9 @@
 name: Release Starwhale base Docker Image
 
 on:
-  workflow_dispatch
+  pull_request:
+    types: [ opened, reopened ]
+  # workflow_dispatch
   #After fixing the rebuild issue, we can enable push option.
   #push:
   #  branches:

.github/workflows/release.yml

@@ -58,67 +58,6 @@ jobs:
           packages_dir: client/dist/
           verify_metadata: false
 
-  starwhale-image-release:
-    runs-on: ubuntu-latest
-    needs:
-      - pypi-release
-
-    strategy:
-      matrix:
-        cuda_version:
-          [
-            "",
-            "11.3",
-            "11.3-cudnn8",
-            "11.4",
-            "11.4-cudnn8",
-            "11.5",
-            "11.5-cudnn8",
-            "11.6",
-            "11.6-cudnn8",
-            "11.7",
-          ]
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get tag
-        id: tag
-        uses: dawidd6/action-get-tag@v1
-        with:
-          strip_v: true
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build and push docker image
-        working-directory: ./docker
-        env:
-          RELEASE_VERSION: ${{ steps.tag.outputs.tag }}
-          PYPI_RELEASE_VERSION: ${{ steps.tag.outputs.tag }}
-        run: |
-          if [ "${{matrix.cuda_version}}" = "" ]; then
-            make build-release-starwhale
-          else
-            make build-release-starwhale-cuda version=${{matrix.cuda_version}}
-          fi
-
   server-image-release:
     runs-on: ubuntu-latest
 

client/scripts/sw-docker-entrypoint

@@ -8,126 +8,13 @@ fi
 
 ulimit -n 65535 || true
 
-CONDA_BIN="/opt/miniconda3/bin"
-PIP_CACHE_DIR=${SW_PIP_CACHE_DIR:=/"${SW_USER:-root}"/.cache/pip}
 VERBOSE="-vvv"
 HANDLER=${SW_RUN_HANDLER:-""}
 STEP=${SW_TASK_STEP:-""}
 TASK_INDEX=${SW_TASK_INDEX:-0}
 TASK_NUM=${SW_TASK_NUM:-0}
 RUNTIME_RESTORED=${SW_USER_RUNTIME_RESTORED:-0}
 
-_update_python_alter() {
-    echo "-->[Preparing] set python/python3 to $1 ..."
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/$1 10
-    update-alternatives --install /usr/bin/python python /usr/bin/$1 10
-    python3 --version
-}
-
-pre_config() {
-    if [ "${SW_TASK_DISABLE_DEBUG}" = "1" ]; then
-        echo "-->[Preparing] debug config ..."
-        VERBOSE="-v"
-    fi
-
-    echo "-->[Preparing] config pypi and conda config ..."
-
-    if [ ${SW_PYPI_INDEX_URL} ] ; then
-        echo -e "\t ** use SW_PYPI_* env to config ~/.pip/pip.conf"
-        mkdir -p ~/.pip
-        cat > ~/.pip/pip.conf << EOF
-[global]
-index-url = ${SW_PYPI_INDEX_URL}
-extra-index-url = ${SW_PYPI_EXTRA_INDEX_URL}
-timeout = ${SW_PYPI_TIMEOUT:-90}
-
-[install]
-trusted-host= ${SW_PYPI_TRUSTED_HOST}
-EOF
-        echo -e "\t ** current pip conf:"
-        echo "-------------------"
-        cat ~/.pip/pip.conf
-        echo "-------------------"
-    else
-        echo -e "\t ** use image builtin pip.conf"
-    fi
-
-    if [ -n "$SW_CONDA_CONFIG" ] ; then
-      echo -e "\t ** use SW_CONDA_CONFIG env to config ~/.condarc"
-      echo "$SW_CONDA_CONFIG" > ~/.condarc
-      echo -e "\t ** current .condarc:"
-      echo "-------------------"
-      cat ~/.condarc
-      echo "-------------------"
-    else
-      echo -e "\t ** use image builtin condarc"
-    fi
-
-}
-
-set_pip_cache() {
-    echo "\t ** set pip cache dir:"
-    python3 -m pip config set global.cache-dir ${PIP_CACHE_DIR} || true
-    python3 -m pip cache dir || true
-}
-
-set_py_and_sw() {
-    pre_config
-
-    echo "-->[Preparing] use runtime ..."
-    _MANIFEST_RUNTIME=$(swcli -o json runtime info ${SW_RUNTIME_VERSION} -of manifest | jq -r ".manifest.environment.python") || exit 1
-    DETECTED_SW_VER=$(swcli -o json runtime info ${SW_RUNTIME_VERSION} -of manifest | jq -r ".manifest.environment.lock.starwhale_version") || exit 1
-
-    _RUNTIME="python${_MANIFEST_RUNTIME}"
-    ORIGIN_SW_VER=$(swcli --version)
-    DETECTED_SW_VER=${DETECTED_SW_VER:-$ORIGIN_SW_VER}
-
-    echo "**** DETECT RUNTIME: python version: ${_RUNTIME}, starwhale version: ${DETECTED_SW_VER}"
-
-    # default python version
-    DEFAULT_PY_VER="python3.8"
-    PY_VER="$DEFAULT_PY_VER"
-    if [ "$_RUNTIME" = "python3.7" ] || [ "$_RUNTIME" = "python3.9" ] || [ "$_RUNTIME" = "python3.10" ] || [ "$_RUNTIME" = "python3.11" ] ; then
-        PY_VER="$_RUNTIME"
-    fi
-    _update_python_alter "$PY_VER"
-
-    set_pip_cache
-
-    if [ "$PY_VER" != "$_RUNTIME" ] || [ "$DETECTED_SW_VER" != "$ORIGIN_SW_VER" ]; then
-        # ignore the editable dev version
-        if [ "$DETECTED_SW_VER" != "0.0.0.dev0" ]; then
-          echo "-->[Preparing] start to install starwhale:${DETECTED_SW_VER}"
-          # install starwhale for current python
-          python3 -m pip install "starwhale==${DETECTED_SW_VER}" || exit 1
-        fi
-    fi
-}
-
-run() {
-    echo "-->[Running] start to run model: ${STEP}, use $(which swcli) cli ..."
-    if [ "${RUNTIME_RESTORED}" != "1" ]; then
-        runtime_args="--runtime ${SW_RUNTIME_VERSION}"
-    else
-        runtime_args="--forbid-packaged-runtime"
-    fi
-
-    swcli ${VERBOSE} model run --handler=${HANDLER} --step=${STEP} \
-        --task-index=${TASK_INDEX} --override-task-num=${TASK_NUM} \
-        --uri ${SW_MODEL_VERSION} --version=${SW_JOB_VERSION} \
-        --forbid-snapshot \
-        ${runtime_args} || exit 1
-}
-
-serve() {
-    echo "-->[Serving] start to serve, use $(which swcli) cli ..."
-    export
-    if [ "${RUNTIME_RESTORED}" != "1" ]; then
-        swcli ${VERBOSE} model serve --uri "${SW_MODEL_VERSION}" --runtime "${SW_RUNTIME_VERSION}" --host 0.0.0.0 || exit 1
-    else
-        swcli ${VERBOSE} model serve --uri "${SW_MODEL_VERSION}" --host 0.0.0.0 --forbid-packaged-runtime || exit 1
-    fi
-}
 
 welcome() {
     echo "===================================="
@@ -148,6 +35,10 @@ welcome() {
 }
 
 prepare(){
+    if [ "${RUNTIME_RESTORED}" == "1" ]; then
+        $(bash "${SW_USER_RUNTIME_WORKDIR:-/opt/starwhale.user/runtime}"/activate.sw)
+    fi
+
     if [ "${SW_INSTANCE_URI}" != "local" ]
     then
         # only remote
@@ -167,10 +58,34 @@ prepare(){
             fi
         fi
     fi
+}
+
+run() {
+    if [ "${SW_TASK_DISABLE_DEBUG}" = "1" ]; then
+        echo "-->[Preparing] debug config ..."
+        VERBOSE="-v"
+    fi
+    echo "-->[Running] start to run model: ${STEP}, use $(which swcli) cli ..."
     if [ "${RUNTIME_RESTORED}" != "1" ]; then
-        set_py_and_sw
+        runtime_args="--runtime ${SW_RUNTIME_VERSION}"
     else
-        $(bash "${SW_USER_RUNTIME_WORKDIR:-/opt/starwhale.user/runtime}"/activate.sw)
+        runtime_args="--forbid-packaged-runtime"
+    fi
+
+    swcli ${VERBOSE} model run --handler=${HANDLER} --step=${STEP} \
+        --task-index=${TASK_INDEX} --override-task-num=${TASK_NUM} \
+        --uri ${SW_MODEL_VERSION} --version=${SW_JOB_VERSION} \
+        --forbid-snapshot \
+        ${runtime_args} || exit 1
+}
+
+serve() {
+    echo "-->[Serving] start to serve, use $(which swcli) cli ..."
+    export
+    if [ "${RUNTIME_RESTORED}" != "1" ]; then
+        swcli ${VERBOSE} model serve --uri "${SW_MODEL_VERSION}" --runtime "${SW_RUNTIME_VERSION}" --host 0.0.0.0 || exit 1
+    else
+        swcli ${VERBOSE} model serve --uri "${SW_MODEL_VERSION}" --host 0.0.0.0 --forbid-packaged-runtime || exit 1
     fi
 }
 
@@ -193,9 +108,6 @@ run_code_server () {
 
 welcome $1
 case "$1" in
-    pre_config)
-        pre_config
-        ;;
     run|evaluation|fine_tune)
         prepare && run
         ;;

client/starwhale/consts/__init__.py

@@ -193,7 +193,7 @@ class FileNode:
 YAML_TYPES = (".yaml", ".yml")
 
 DEFAULT_IMAGE_REPO = "docker-registry.starwhale.cn/star-whale"
-DEFAULT_IMAGE_NAME = "starwhale"
+DEFAULT_IMAGE_NAME = "base"
 SW_IMAGE_FMT = "{repo}/{name}:{tag}"
 DEFAULT_SW_TASK_RUN_IMAGE = SW_IMAGE_FMT.format(
     repo=DEFAULT_IMAGE_REPO, name=DEFAULT_IMAGE_NAME, tag=LATEST_TAG

client/starwhale/core/runtime/model.py

@@ -1850,6 +1850,7 @@ def _render_dockerfile(_manifest: t.Dict[str, t.Any]) -> None:
                 pypi_extra_index_url=" ".join(_pip.get("extra_index_url", [])),
                 pypi_trusted_host=" ".join(_pip.get("trusted_host", [])),
                 python_version=_manifest["environment"]["python"],
+                starwhale_version=_manifest["environment"]["lock"]["starwhale_version"],
                 mode=_manifest["environment"]["mode"],
                 local_packaged_env=_manifest["dependencies"].get(
                     "local_packaged_env", False

client/starwhale/core/runtime/template/Dockerfile.tmpl

@@ -15,14 +15,10 @@ COPY . /opt/starwhale.user/runtime
 ENV SW_PYPI_INDEX_URL "{{pypi_index_url}}"
 ENV SW_PYPI_EXTRA_INDEX_URL "{{pypi_extra_index_url}}"
 ENV SW_PYPI_TRUSTED_HOST "{{pypi_trusted_host}}"
-RUN sw-docker-entrypoint pre_config || true
-
-ENV SW_USER_RUNTIME_PYTHON_VERSION={{python_version}}
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${SW_USER_RUNTIME_PYTHON_VERSION} 10 \
-    && update-alternatives --install /usr/bin/python python /usr/bin/python${SW_USER_RUNTIME_PYTHON_VERSION} 10
-
 ENV SW_PIP_CACHE_DIR=/root/.cache/pip
-RUN python3 -m pip config set global.cache-dir ${SW_PIP_CACHE_DIR}
+ENV SW_RUNTIME_PYTHON_VERSION={{python_version}}
+ENV SW_VERSION={{starwhale_version}}
+RUN entrypoint set_py_and_sw || true
 
 RUN swcli -vvv runtime restore /opt/starwhale.user/runtime \
     && $(bash /opt/starwhale.user/runtime/activate.sw)

docker/Dockerfile.base

@@ -8,6 +8,12 @@ ENV LANG=en_US.UTF-8
 ENV LC_ALL=en_US.UTF-8
 ENV TZ=Etc/UTC
 
+ENV SW_CONTAINER=1
+ENV TARGETARCH=${TARGETARCH}
+ENV COLUMNS=250
+
+SHELL ["/bin/bash", "-c"]
+
 COPY external/render_sources_list.sh /tmp/render_sources_list.sh
 RUN bash /tmp/render_sources_list.sh ${TARGETARCH}
 
@@ -43,6 +49,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
     && bash /tmp/conda.sh -b -p /opt/miniconda3 \
     && /opt/miniconda3/bin/conda clean -a -y -f \
     && rm -rf /tmp/conda.sh
+COPY external/condarc /root/.condarc
 
 RUN python3.7 -m pip install --upgrade virtualenv>=20.14.0 pip>=22.1.0 \
     && python3.8 -m pip install --upgrade virtualenv>=20.14.0 pip>=22.1.0 \
@@ -52,3 +59,11 @@ RUN python3.7 -m pip install --upgrade virtualenv>=20.14.0 pip>=22.1.0 \
     && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
     && python3.11 -m pip install --upgrade virtualenv>=20.14.0 pip>=22.1.0 \
     && python3 -m pip install yq && python3 -m pip cache purge
+
+# code-server
+RUN curl -fsSL --compressed  https://code-server.dev/install.sh | sh
+
+# stable entrypoint
+WORKDIR /opt/starwhale/swmp
+COPY external/starwhale-entrypoint.sh entrypoint
+ENTRYPOINT ["entrypoint"]