diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 8bdfc606d..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: 'Publish CML dockers' - -on: [push, pull_request] - -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }} - TEST_GITHUB_REPO: https://github.com/iterative/cml_qa_tests_dummy - TEST_GITHUB_SHA: 62edc8b3f46a60b3fe1e5c08fd3e0046d350ee29 - TEST_GITLAB_TOKEN: ${{ secrets.TEST_GITLAB_TOKEN }} - TEST_GITLAB_REPO: https://gitlab.com/iterative.ai/cml_qa_tests_dummy - TEST_GITLAB_SHA: c4c13286e78dc252dd2611f31a755f10d343fbd4 - TEST_BBCLOUD_TOKEN: ${{ secrets.TEST_BBCLOUD_TOKEN }} - TEST_BBCLOUD_REPO: https://bitbucket.org/iterative-ai/cml-qa-tests-dummy - TEST_BBCLOUD_SHA: 9bb9131ce0af294fe1c6eedca1f2bce3983e80bd - -jobs: - test_and_deploy: - runs-on: [ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - - - name: 'npm ci' - run: npm ci - - - name: 'lint' - run: npm run lint - - - name: 'tests' - run: | - sudo update-alternatives --install /usr/bin/python python $(which python3) 10 - sudo apt-get update && sudo apt-get install -y python3-pip - sudo pip install --upgrade pip - sudo pip install --upgrade setuptools - sudo pip install tensorboard - - npm run test - - - name: Publish CML docker image - # only publish if push to master (dvcorg/cml:latest) - # or create a tag in the repo (dvcorg/cml:tag) - if: - github.event_name == 'push' && (contains(github.ref, 'tags') || - github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: dvcorg/cml - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile - context: ./ - cache: true - tag_names: true - - - name: Publish cml-py3 docker image - if: - github.event_name == 'push' && (contains(github.ref, 'tags') || - github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - env: - DOCKER_FROM: cml - with: - name: dvcorg/cml-py3 - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile-py3 - context: ./ - cache: true - tag_names: true - buildargs: DOCKER_FROM - - - name: Publish cml-dev docker image - if: - github.event_name == 'push' && (contains(github.ref, 'tags') || - github.ref == 'refs/heads/master') - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: dvcorg/cml-dev - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - dockerfile: ./docker/Dockerfile-dev - context: ./ - cache: true - tag_names: true - - # cloud-runner aliases will be deprecated - - name: Alias cloud-runner - if: - github.event_name == 'push' && (contains(github.ref, 'tags') || - github.ref == 'refs/heads/master') - run: | - echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - - docker tag dvcorg/cml dvcorg/cml-gpu && docker push dvcorg/cml-gpu - docker tag dvcorg/cml-py3 dvcorg/cml-gpu-py3 && docker push dvcorg/cml-gpu-py3 - - docker tag dvcorg/cml dvcorg/cml-cloud-runner && docker push dvcorg/cml-cloud-runner - docker tag dvcorg/cml-py3 dvcorg/cml-py3-cloud-runner && docker push dvcorg/cml-py3-cloud-runner - docker tag dvcorg/cml-gpu dvcorg/cml-gpu-cloud-runner && docker push dvcorg/cml-gpu-cloud-runner - docker tag dvcorg/cml-gpu-py3 dvcorg/cml-gpu-py3-cloud-runner && docker push dvcorg/cml-gpu-py3-cloud-runner - - - name: Publish to NPM - if: - github.event_name == 'push' && (contains(github.ref, 'tags') || - github.ref == 'refs/heads/master') - run: | - npm config set //registry.npmjs.org/:_authToken=$NPM_AUTH_TOKEN - npm publish - env: - NPM_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..6f58adea3 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,76 @@ +name: Release packages and images + +on: + workflow_dispatch: + release: + types: + - published + - edited + +jobs: + package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: npm install + - run: npm publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + docker: + strategy: + matrix: + cml: [0] + dvc: [1, 2] + gpu: [false, true] + base: [0, 1] + include: + - base: 0 + cuda: 10.1 + cudnn: 7 + python: 2.7 + ubuntu: 18.04 + - base: 1 + cuda: 11.0.3 + cudnn: 8 + python: 3.8 + ubuntu: 20.04 + runs-on: ubuntu-latest + needs: package + + steps: + - uses: actions/checkout@v2 + - uses: docker/setup-buildx-action@v1 + - uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + - uses: docker/build-push-action@v2 + if: matrix.gpu == false + with: + push: true + context: ./ + file: ./Dockerfile + tags: + dvcorg/cml:${{ matrix.cml }}-dvc${{ matrix.dvc }}-base${{ + matrix.base }} + build-args: | + CML_VERSION=${{ matrix.cml }} + DVC_VERSION=${{ matrix.dvc }} + PYTHON_VERSION=${{ matrix.python }} + BASE_IMAGE=ubuntu:${{ matrix.ubuntu }} + + - uses: docker/build-push-action@v2 + if: matrix.gpu == true + with: + push: true + context: ./ + file: ./Dockerfile + tags: + dvcorg/cml:${{ matrix.cml }}-dvc${{ matrix.dvc }}-base${{ + matrix.base }}-gpu + build-args: | + CML_VERSION=${{ matrix.cml }} + DVC_VERSION=${{ matrix.dvc }} + PYTHON_VERSION=${{ matrix.python }} + BASE_IMAGE=nvidia/cuda:${{ matrix.cuda }}-cudnn${{ matrix.cudnn }}-runtime-ubuntu${{ matrix.ubuntu }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..1f99d5bca --- /dev/null +++ b/Dockerfile @@ -0,0 +1,104 @@ +ARG BASE_IMAGE=ubuntu:20.04 +FROM ${BASE_IMAGE} + +# TODO: consider using iterative.ai or something else +LABEL maintainer="dvcorg " + +# CONFIGURE NON-INTERACTIVE APT +ENV DEBIAN_FRONTEND=noninteractive +RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes + +# CONFIGURE SHELL +SHELL ["/bin/bash", "-c"] + +# INSTALL CORE DEPENDENCIES +RUN apt-get update \ + && apt-get install --no-install-recommends \ + build-essential \ + apt-utils \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + pkg-config \ + curl \ + wget \ + unzip \ + gpg-agent \ + sudo \ + tzdata \ + locales \ + && locale-gen en_US.UTF-8 \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + +# CONFIGURE LOCALE +ENV LANG="en_US.UTF-8" +ENV LANGUAGE="en_US:en" +ENV LC_ALL="en_US.UTF-8" + +# INSTALL NODE, GIT & GO +RUN add-apt-repository ppa:git-core/ppa --yes \ + && add-apt-repository ppa:longsleep/golang-backports --yes \ + && curl --location https://deb.nodesource.com/setup_12.x | bash \ + && apt-get update \ + && apt-get install --yes git golang-go nodejs \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + +# INSTALL TERRAFORM +RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - \ + && apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release --codename --short) main" \ + && apt update \ + && apt-get install --yes terraform \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + +# INSTALL PYTHON +ARG PYTHON_VERSION=3 +RUN add-apt-repository universe --yes \ + && apt-get update \ + && PYTHON_SUFFIX="$(sed --expression='s/3.*/3/g' --expression='s/2.*//g' <<< "${PYTHON_VERSION}")" \ + && apt-get install --yes --no-install-recommends python${PYTHON_VERSION} python${PYTHON_SUFFIX}{-pip,-setuptools} \ + && update-alternatives --install /usr/bin/python python${PYTHON_VERSION} $(which python${PYTHON_VERSION}) 10 \ + && python -m pip install pip --upgrade \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + +# INSTALL DVC +ARG DVC_VERSION=2 +RUN cd /etc/apt/sources.list.d \ + && wget https://dvc.org/deb/dvc.list \ + && apt-get update \ + && apt-get install --yes "dvc=${DVC_VERSION}.*" \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + +# INSTALL CML +ARG CML_VERSION=0 +RUN npm config set user 0 \ + && npm install --global "git+https://github.com/iterative/cml" + +# INSTALL VEGA +RUN add-apt-repository universe --yes \ + && apt-get update \ + && apt-get install --yes \ + libcairo2-dev \ + libpango1.0-dev \ + libjpeg-dev \ + libgif-dev \ + librsvg2-dev \ + libfontconfig-dev \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* \ + && npm config set user 0 \ + && npm install --global canvas vega vega-cli vega-lite + +# CONFIGURE RUNNER PATH +ENV RUNNER_PATH=/home/runner +ENV RUNNER_ALLOW_RUNASROOT=1 +RUN mkdir ${RUNNER_PATH} +WORKDIR ${RUNNER_PATH} + +# COMMAND +ENV IN_DOCKER=1 +CMD ["cml-runner"] diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index dba8edc04..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,115 +0,0 @@ -ARG ARCH= -ARG CUDA=10.1 -ARG UBUNTU_VERSION=18.04 - -FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base - -LABEL maintainer="dvc.org" - -ENV DEBIAN_FRONTEND=noninteractive -RUN echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes - -RUN apt-get update && \ - apt-get install --no-install-recommends --fix-missing \ - build-essential \ - apt-utils \ - apt-transport-https \ - ca-certificates \ - software-properties-common \ - pkg-config \ - curl \ - wget \ - unzip \ - gpg-agent \ - sudo \ - tzdata \ - locales && \ - locale-gen en_US.UTF-8 && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# NODE GIT DVC VEGA DEPS -RUN add-apt-repository universe -y && \ - add-apt-repository ppa:git-core/ppa -y && \ - add-apt-repository ppa:longsleep/golang-backports -y && \ - curl -sL https://deb.nodesource.com/setup_12.x | bash && \ - apt-get update && \ - apt-get install -y \ - git nodejs \ - libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev libfontconfig-dev \ - golang-go && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN npm config set user 0 && \ - npm install -g canvas vega vega-cli vega-lite - -ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' - -# CUDNN -ARG CUDA -ARG CUDNN=7.6.4.38-1 -ARG CUDNN_MAJOR=7 -ARG LIBNVINFER=6.0.1-1 -ARG LIBNVINFER_MAJOR=6 -ARG CUBLAS=10.2.1.243-1 -ARG CUBLAS_MAJOR=10 - -SHELL ["/bin/bash", "-c"] - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cuda-command-line-tools-${CUDA/./-} \ - libcublas${CUBLAS_MAJOR}=${CUBLAS} \ - cuda-nvrtc-${CUDA/./-} \ - cuda-cufft-${CUDA/./-} \ - cuda-curand-${CUDA/./-} \ - cuda-cusolver-${CUDA/./-} \ - cuda-cusparse-${CUDA/./-} \ - libcudnn${CUDNN_MAJOR}=${CUDNN}+cuda${CUDA} \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH - -# TENSORFLOW -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - libnvinfer${LIBNVINFER_MAJOR}=${LIBNVINFER}+cuda${CUDA} \ - libnvinfer-plugin${LIBNVINFER_MAJOR}=${LIBNVINFER}+cuda${CUDA} && \ - ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ - echo /usr/local/cuda/lib64/stubs > /etc/ld.so.conf.d/z-cuda-stubs.conf && ldconfig && \ - apt-get clean && rm -rf /var/lib/apt/lists/* -# TENSORFLOW ENDS - -# DOCKER, OUR CUSTOM DOCKER MACHINE FORK THAT SUPPORTS GPU ACCELERATOR (DEPRECATED) -ENV GO111MODULE=auto -RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \ - go get github.com/iterative/machine && \ - mv $(go env GOPATH)/src/github.com/iterative $(go env GOPATH)/src/github.com/docker && \ - cd $(go env GOPATH)/src/github.com/docker/machine && make build && \ - ln -s $(go env GOPATH)/src/github.com/docker/machine/bin/docker-machine /usr/local/bin/docker-machine && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# TERRAFORM -RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && \ - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" && \ - apt update && apt-get install -y terraform && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# GITLAB RUNNER AND GITHUB RUNNER -ENV RUNNER_PATH=/home/runner -ENV RUNNER_ALLOW_RUNASROOT=1 - -RUN mkdir ${RUNNER_PATH} -WORKDIR ${RUNNER_PATH} - -# CML -ADD "./" "/cml" -RUN npm config set user 0 && \ - npm install -g /cml - -RUN wget https://dvc.org/deb/dvc.list -O /etc/apt/sources.list.d/dvc.list && \ - apt-get update && apt-get install -y dvc && apt-get clean && rm -rf /var/lib/apt/lists/* - -ENV IN_DOCKER=1 -CMD ["cml-runner"] diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev deleted file mode 100644 index f3d448585..000000000 --- a/docker/Dockerfile-dev +++ /dev/null @@ -1,17 +0,0 @@ -FROM dvcorg/cml-py3:latest - -LABEL maintainer="dvc.org" - -RUN apt-get remove -y dvc && \ - apt-get update && \ - apt-get install -y --no-install-recommends --fix-missing \ - git \ - python3-setuptools \ - python3-dev && \ - git clone https://github.com/iterative/dvc/ && \ - cd dvc && \ - # git checkout 128fa7f && \ - pip install -U -e .[all,tests] && \ - cd .. && \ - rm -rf dvc - diff --git a/docker/Dockerfile-py3 b/docker/Dockerfile-py3 deleted file mode 100644 index 846d9d1a1..000000000 --- a/docker/Dockerfile-py3 +++ /dev/null @@ -1,15 +0,0 @@ -ARG DOCKER_FROM=cml - -FROM dvcorg/${DOCKER_FROM}:latest as base - -LABEL maintainer="dvc.org" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends --fix-missing \ - python3 \ - python3-pip \ - python3-setuptools && \ - pip3 install --upgrade pip && \ - update-alternatives --install \ - /usr/bin/python python $(which python3) 10 && \ - apt-get clean && rm -rf /var/lib/apt/lists/*