From 952e4a137a087a4078c8e4b0c94747e4f6b82720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eahin?= Date: Thu, 13 May 2021 16:56:18 +0300 Subject: [PATCH 01/22] Split push from build-all and added Dockerfile generator --- .gitignore | 1 + build-all.bash | 13 ++++++++----- generate-all.bash | 30 ++++++++++++++++++++++++++++++ push-all.bash | 22 ++++++++++++++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100755 generate-all.bash create mode 100644 push-all.bash diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d74e21 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/ diff --git a/build-all.bash b/build-all.bash index eecf601..cacc018 100755 --- a/build-all.bash +++ b/build-all.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script builds all Docker containers described by Dockerfiles in this repository @@ -15,9 +15,14 @@ set -vex -RELEASE_HASH=$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8) +HERE="$( cd "$(dirname "$0")" ; pwd -P )" + +export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" +export TAGPREFIX="${TAGPREFIX:-dvcorg}" + + +./generate-all.bash -TAGPREFIX=dvcorg DIR=$(dirname $0) cd "${DIR}" @@ -31,6 +36,4 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do fi echo "BUILDING: ${dockerdir} with the tag: ${TAGPREFIX}/${TAG}" docker build --build-arg RELEASE_HASH=${RELEASE_HASH} -t ${TAGPREFIX}/${TAG} ${dockerdir}/ - echo "PUSHING: ${dockerdir} with the tag: ${TAGPREFIX}/${TAG}" - docker push ${TAGPREFIX}/${TAG} done diff --git a/generate-all.bash b/generate-all.bash new file mode 100755 index 0000000..ebb7905 --- /dev/null +++ b/generate-all.bash @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -vex + +HERE="$( cd "$(dirname "$0")" ; pwd -P )" +export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" +export TAGPREFIX="${TAGPREFIX:-dvcorg}" + +DIR=$(dirname $0) +cd "${DIR}" + +find ${DIR} -name generate.* | sort | while read -r filepath ; do + filename=$(basename -- "$filepath") + dirname=$(dirname -- "$filepath") + ext="${filename##*.}" + cd $dirname + if [[ "$ext" == "bash" ]] ; then + bash "$filename" + elif [[ "$ext" == "sh" ]] ; then + sh "$filename" + elif [[ "$ext" == "zsh" ]] ; then + zsh "$filename" + elif [[ "$ext" == "py" ]] ; then + python "$filename" + else + echo "UNSUPPORTED GENERATOR SCRIPT: $filepath" + fi + cd - +done + diff --git a/push-all.bash b/push-all.bash new file mode 100644 index 0000000..3c22977 --- /dev/null +++ b/push-all.bash @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -vex + +HERE="$( cd "$(dirname "$0")" ; pwd -P )" +export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" +export TAGPREFIX="${TAGPREFIX:-dvcorg}" + +DIR=$(dirname $0) +cd "${DIR}" + +find $DIR -name Dockerfile | sort | while read -r filepath ; do + dockerdir=$(dirname ${filepath}) + tagfile=${dockerdir}/Dockertag + if [ -f ${tagfile} ] ; then + TAG=$(head -n 1 ${tagfile}) + else + TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') + fi + echo "PUSHING: ${dockerdir} with the tag: ${TAGPREFIX}/${TAG}" + docker push ${TAGPREFIX}/${TAG} +done From 5cbd699c8bf038720e6cc1846f313e40d134614d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eahin?= Date: Thu, 13 May 2021 16:58:00 +0300 Subject: [PATCH 02/22] added Dockerfile generation for dvc-get-started --- start-script/.gitignore | 2 ++ start-script/Dockerfile.template | 29 ++++++++++++++++ start-script/Dockertag.template | 1 + start-script/bashrc | 57 ++++++++++++++++++++++++++++++++ start-script/generate.bash | 30 +++++++++++++++++ 5 files changed, 119 insertions(+) create mode 100644 start-script/.gitignore create mode 100644 start-script/Dockerfile.template create mode 100644 start-script/Dockertag.template create mode 100644 start-script/bashrc create mode 100755 start-script/generate.bash diff --git a/start-script/.gitignore b/start-script/.gitignore new file mode 100644 index 0000000..152266f --- /dev/null +++ b/start-script/.gitignore @@ -0,0 +1,2 @@ +dvc-get-started/ +build/ diff --git a/start-script/Dockerfile.template b/start-script/Dockerfile.template new file mode 100644 index 0000000..f7f0af5 --- /dev/null +++ b/start-script/Dockerfile.template @@ -0,0 +1,29 @@ +FROM python:3.8 + +WORKDIR /root + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN RELEASE_HASH=${RELEASE_HASH} \ + apt-get update -y && apt-get install -y \ + gnupg \ + wget \ + tree + +RUN RELEASE_HASH=${RELEASE_HASH} \ + pip install 'dvc[all]' + +COPY dvc-get-started dvc-get-started +COPY .bashrc .bashrc + +# Configure git user name and email +RUN git config --global user.email "guest@example.com" \ + && git config --global user.name "Guest User" + +CMD ["/bin/bash", "-i"] + + + + + diff --git a/start-script/Dockertag.template b/start-script/Dockertag.template new file mode 100644 index 0000000..0255d2a --- /dev/null +++ b/start-script/Dockertag.template @@ -0,0 +1 @@ +dvcorg/doc-start:${GIT_TAG} diff --git a/start-script/bashrc b/start-script/bashrc new file mode 100644 index 0000000..b2939fc --- /dev/null +++ b/start-script/bashrc @@ -0,0 +1,57 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +# don't put duplicate lines in the history. See bash(1) for more options +# don't overwrite GNU Midnight Commander's setting of `ignorespace'. +HISTCONTROL=$HISTCONTROL${HISTCONTROL+:}ignoredups +# ... or force ignoredups and ignorespace +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# make less more friendly for non-text input files, see lesspipe(1) +#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# prompt +PS1='\[\033[01;34m\]\w\[\033[00m\]$ \[\033[01;32m\]' +trap 'echo -ne "\033[00m"' DEBUG + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + alias dir='dir --color=auto' + alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# some more ls aliases +alias ll='ls -l' +alias la='ls -A' +alias l='ls -CF' + +# enable dvc completion +dvc completion -s bash > /etc/bash_completion.d/dvc + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if [ -f /etc/bash_completion ] && ! shopt -oq posix; then + . /etc/bash_completion +fi + +if [ -f $HOME/start.sh ] ; then + $HOME/start.sh +fi diff --git a/start-script/generate.bash b/start-script/generate.bash new file mode 100755 index 0000000..fc815c8 --- /dev/null +++ b/start-script/generate.bash @@ -0,0 +1,30 @@ +#!/bin/bash + + +HERE="$( cd "$(dirname "$0")" ; pwd -P )" + +if [[ -d build ]] ; then + echo "Please delete ${PWD}/build directory first." + exit +fi + +# Clone dvc-get-started +git clone https://github.com/iterative/dvc-get-started + +GIT_TAGS=$(git -C dvc-get-started tag --list) + +for TAG in ${GIT_TAGS} ; do + TAG_DIR="build/${TAG}" + mkdir -p ${TAG_DIR} + + export GIT_TAG=${TAG} + + git clone dvc-get-started -b ${GIT_TAG} ${TAG_DIR}/dvc-get-started + + cp bashrc ${TAG_DIR}/.bashrc + cat Dockerfile.template | envsubst > ${TAG_DIR}/Dockerfile + cat Dockertag.template | envsubst > ${TAG_DIR}/Dockertag +done + +unset GIT_TAG +unset RELEASE_HASH From cf0db6b7399b5900547dade432da80129c54066d Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Thu, 13 May 2021 15:11:23 +0000 Subject: [PATCH 03/22] Renamed TAGPREFIX to TAG_PREFIX --- build-all.bash | 19 +++---------------- generate-all.bash | 2 +- push-all.bash | 11 ++++++++--- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/build-all.bash b/build-all.bash index cacc018..c8c68e5 100755 --- a/build-all.bash +++ b/build-all.bash @@ -1,24 +1,11 @@ #!/usr/bin/env bash - -# This script builds all Docker containers described by Dockerfiles in this repository -# For each Dockerfile it runs: -# $ docker build -t $TAGPREFIX/$TAG -# and -# $ docker push -t $TAGPREFIX/$TAG -# -# $TAGPREFIX is defined in this file. -# If a Dockerfile has a Dockertag file in the same directory, it's read from there -# Otherwise $TAG is generated by replacing / and with - in the directory name. -# -# docker build command uses --no-cache option to prevent stale layers to be used. - set -vex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" -export TAGPREFIX="${TAGPREFIX:-dvcorg}" +export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" ./generate-all.bash @@ -34,6 +21,6 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do else TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') fi - echo "BUILDING: ${dockerdir} with the tag: ${TAGPREFIX}/${TAG}" - docker build --build-arg RELEASE_HASH=${RELEASE_HASH} -t ${TAGPREFIX}/${TAG} ${dockerdir}/ + echo "BUILDING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" + docker build --build-arg RELEASE_HASH=${RELEASE_HASH} -t ${TAG_PREFIX}/${TAG} ${dockerdir}/ done diff --git a/generate-all.bash b/generate-all.bash index ebb7905..4e9889a 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -4,7 +4,7 @@ set -vex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" -export TAGPREFIX="${TAGPREFIX:-dvcorg}" +export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" DIR=$(dirname $0) cd "${DIR}" diff --git a/push-all.bash b/push-all.bash index 3c22977..bda71ab 100644 --- a/push-all.bash +++ b/push-all.bash @@ -4,7 +4,7 @@ set -vex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" -export TAGPREFIX="${TAGPREFIX:-dvcorg}" +export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" DIR=$(dirname $0) cd "${DIR}" @@ -17,6 +17,11 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do else TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') fi - echo "PUSHING: ${dockerdir} with the tag: ${TAGPREFIX}/${TAG}" - docker push ${TAGPREFIX}/${TAG} + if [[ "${TAG}" ~= '.*/.*' ]] ; then + echo "PUSHING: ${dockerdir} with the tag: ${TAG}" + docker push ${TAG} + else + echo "PUSHING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" + docker push ${TAG_PREFIX}/${TAG} + fi done From cc248de8fa75a3e88b6f8062b1735e8d2aa42ebf Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Thu, 13 May 2021 15:11:49 +0000 Subject: [PATCH 04/22] moved start-script to start --- start-script/bashrc | 57 --------------------- {start-script => start}/.gitignore | 0 start/Dockerfile | 36 ------------- {start-script => start}/Dockerfile.template | 0 start/Dockertag | 1 - {start-script => start}/Dockertag.template | 0 {start-script => start}/generate.bash | 0 7 files changed, 94 deletions(-) delete mode 100644 start-script/bashrc rename {start-script => start}/.gitignore (100%) delete mode 100644 start/Dockerfile rename {start-script => start}/Dockerfile.template (100%) delete mode 100644 start/Dockertag rename {start-script => start}/Dockertag.template (100%) rename {start-script => start}/generate.bash (100%) diff --git a/start-script/bashrc b/start-script/bashrc deleted file mode 100644 index b2939fc..0000000 --- a/start-script/bashrc +++ /dev/null @@ -1,57 +0,0 @@ -# ~/.bashrc: executed by bash(1) for non-login shells. - -# If not running interactively, don't do anything -[ -z "$PS1" ] && return - -# don't put duplicate lines in the history. See bash(1) for more options -# don't overwrite GNU Midnight Commander's setting of `ignorespace'. -HISTCONTROL=$HISTCONTROL${HISTCONTROL+:}ignoredups -# ... or force ignoredups and ignorespace -HISTCONTROL=ignoreboth - -# append to the history file, don't overwrite it -shopt -s histappend - -# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) - -# check the window size after each command and, if necessary, -# update the values of LINES and COLUMNS. -shopt -s checkwinsize - -# make less more friendly for non-text input files, see lesspipe(1) -#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" - -# prompt -PS1='\[\033[01;34m\]\w\[\033[00m\]$ \[\033[01;32m\]' -trap 'echo -ne "\033[00m"' DEBUG - -# enable color support of ls and also add handy aliases -if [ -x /usr/bin/dircolors ]; then - test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" - alias ls='ls --color=auto' - alias dir='dir --color=auto' - alias vdir='vdir --color=auto' - - alias grep='grep --color=auto' - alias fgrep='fgrep --color=auto' - alias egrep='egrep --color=auto' -fi - -# some more ls aliases -alias ll='ls -l' -alias la='ls -A' -alias l='ls -CF' - -# enable dvc completion -dvc completion -s bash > /etc/bash_completion.d/dvc - -# enable programmable completion features (you don't need to enable -# this, if it's already enabled in /etc/bash.bashrc and /etc/profile -# sources /etc/bash.bashrc). -if [ -f /etc/bash_completion ] && ! shopt -oq posix; then - . /etc/bash_completion -fi - -if [ -f $HOME/start.sh ] ; then - $HOME/start.sh -fi diff --git a/start-script/.gitignore b/start/.gitignore similarity index 100% rename from start-script/.gitignore rename to start/.gitignore diff --git a/start/Dockerfile b/start/Dockerfile deleted file mode 100644 index 3b8ef2d..0000000 --- a/start/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:20.04 - -WORKDIR /root - -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 - -ARG RELEASE_HASH -RUN RELEASE_HASH=${RELEASE_HASH} \ - apt-get update -y && apt-get install -y \ - gnupg \ - wget \ - tree - -RUN wget \ - https://dvc.org/deb/dvc.list \ - -O /etc/apt/sources.list.d/dvc.list - -RUN wget -qO - https://dvc.org/deb/iterative.asc | apt-key add - - -RUN RELEASE_HASH=${RELEASE_HASH} \ - apt-get update -y && apt-get install -y \ - dvc - -COPY bashrc .bashrc - -# Configure git user name and email -RUN git config --global user.email "guest@example.com" \ - && git config --global user.name "Guest User" - -CMD ["/bin/bash", "-i"] - - - - - diff --git a/start-script/Dockerfile.template b/start/Dockerfile.template similarity index 100% rename from start-script/Dockerfile.template rename to start/Dockerfile.template diff --git a/start/Dockertag b/start/Dockertag deleted file mode 100644 index 14e6736..0000000 --- a/start/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-start:base diff --git a/start-script/Dockertag.template b/start/Dockertag.template similarity index 100% rename from start-script/Dockertag.template rename to start/Dockertag.template diff --git a/start-script/generate.bash b/start/generate.bash similarity index 100% rename from start-script/generate.bash rename to start/generate.bash From bf9d59c576258ae0f8f1bcbbc361ecf2b33a076b Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Thu, 13 May 2021 15:14:54 +0000 Subject: [PATCH 05/22] updated katacoda docker generation to be script based --- katacoda/.gitignore | 2 + katacoda/{Dockerfile => Dockerfile.template} | 1 - katacoda/Dockertag | 1 - katacoda/Dockertag.template | 1 + katacoda/base/Dockerfile.template | 35 ++++++++++++ katacoda/base/Dockertag.template | 1 + katacoda/base/bashrc | 57 +++++++++++++++++++ katacoda/generate.bash | 25 ++++++++ .../{Dockerfile => Dockerfile.template} | 2 +- katacoda/get-started/01-initialize/Dockertag | 1 - .../01-initialize/Dockertag.template | 1 + .../{Dockerfile => Dockerfile.template} | 2 +- katacoda/get-started/02-versioning/Dockertag | 1 - .../02-versioning/Dockertag.template | 1 + .../{Dockerfile => Dockerfile.template} | 3 +- katacoda/get-started/03-accessing/Dockertag | 1 - .../03-accessing/Dockertag.template | 1 + .../{Dockerfile => Dockerfile.template} | 4 +- katacoda/get-started/04-stages/Dockertag | 1 - .../get-started/04-stages/Dockertag.template | 1 + .../{Dockerfile => Dockerfile.template} | 3 +- .../05-params-metrics-plots/Dockertag | 1 - .../Dockertag.template | 1 + .../{Dockerfile => Dockerfile.template} | 6 +- katacoda/get-started/06-experiments/Dockertag | 1 - .../06-experiments/Dockertag.template | 1 + 26 files changed, 133 insertions(+), 22 deletions(-) create mode 100644 katacoda/.gitignore rename katacoda/{Dockerfile => Dockerfile.template} (97%) delete mode 100644 katacoda/Dockertag create mode 100644 katacoda/Dockertag.template create mode 100644 katacoda/base/Dockerfile.template create mode 100644 katacoda/base/Dockertag.template create mode 100644 katacoda/base/bashrc create mode 100755 katacoda/generate.bash rename katacoda/get-started/01-initialize/{Dockerfile => Dockerfile.template} (85%) delete mode 100644 katacoda/get-started/01-initialize/Dockertag create mode 100644 katacoda/get-started/01-initialize/Dockertag.template rename katacoda/get-started/02-versioning/{Dockerfile => Dockerfile.template} (88%) delete mode 100644 katacoda/get-started/02-versioning/Dockertag create mode 100644 katacoda/get-started/02-versioning/Dockertag.template rename katacoda/get-started/03-accessing/{Dockerfile => Dockerfile.template} (90%) delete mode 100644 katacoda/get-started/03-accessing/Dockertag create mode 100644 katacoda/get-started/03-accessing/Dockertag.template rename katacoda/get-started/04-stages/{Dockerfile => Dockerfile.template} (94%) delete mode 100644 katacoda/get-started/04-stages/Dockertag create mode 100644 katacoda/get-started/04-stages/Dockertag.template rename katacoda/get-started/05-params-metrics-plots/{Dockerfile => Dockerfile.template} (94%) delete mode 100644 katacoda/get-started/05-params-metrics-plots/Dockertag create mode 100644 katacoda/get-started/05-params-metrics-plots/Dockertag.template rename katacoda/get-started/06-experiments/{Dockerfile => Dockerfile.template} (84%) delete mode 100644 katacoda/get-started/06-experiments/Dockertag create mode 100644 katacoda/get-started/06-experiments/Dockertag.template diff --git a/katacoda/.gitignore b/katacoda/.gitignore new file mode 100644 index 0000000..152266f --- /dev/null +++ b/katacoda/.gitignore @@ -0,0 +1,2 @@ +dvc-get-started/ +build/ diff --git a/katacoda/Dockerfile b/katacoda/Dockerfile.template similarity index 97% rename from katacoda/Dockerfile rename to katacoda/Dockerfile.template index 7c4b24e..dd12825 100644 --- a/katacoda/Dockerfile +++ b/katacoda/Dockerfile.template @@ -5,7 +5,6 @@ WORKDIR /root ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 -ARG RELEASE_HASH RUN RELEASE_HASH=${RELEASE_HASH} \ apt-get update -y && apt-get install -y \ gnupg \ diff --git a/katacoda/Dockertag b/katacoda/Dockertag deleted file mode 100644 index fbe2853..0000000 --- a/katacoda/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:base diff --git a/katacoda/Dockertag.template b/katacoda/Dockertag.template new file mode 100644 index 0000000..fea98e1 --- /dev/null +++ b/katacoda/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:base diff --git a/katacoda/base/Dockerfile.template b/katacoda/base/Dockerfile.template new file mode 100644 index 0000000..dd12825 --- /dev/null +++ b/katacoda/base/Dockerfile.template @@ -0,0 +1,35 @@ +FROM ubuntu:20.04 + +WORKDIR /root + +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +RUN RELEASE_HASH=${RELEASE_HASH} \ + apt-get update -y && apt-get install -y \ + gnupg \ + wget \ + tree + +RUN wget \ + https://dvc.org/deb/dvc.list \ + -O /etc/apt/sources.list.d/dvc.list + +RUN wget -qO - https://dvc.org/deb/iterative.asc | apt-key add - + +RUN RELEASE_HASH=${RELEASE_HASH} \ + apt-get update -y && apt-get install -y \ + dvc + +COPY bashrc .bashrc + +# Configure git user name and email +RUN git config --global user.email "guest@example.com" \ + && git config --global user.name "Guest User" + +CMD ["/bin/bash", "-i"] + + + + + diff --git a/katacoda/base/Dockertag.template b/katacoda/base/Dockertag.template new file mode 100644 index 0000000..fea98e1 --- /dev/null +++ b/katacoda/base/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:base diff --git a/katacoda/base/bashrc b/katacoda/base/bashrc new file mode 100644 index 0000000..b2939fc --- /dev/null +++ b/katacoda/base/bashrc @@ -0,0 +1,57 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. + +# If not running interactively, don't do anything +[ -z "$PS1" ] && return + +# don't put duplicate lines in the history. See bash(1) for more options +# don't overwrite GNU Midnight Commander's setting of `ignorespace'. +HISTCONTROL=$HISTCONTROL${HISTCONTROL+:}ignoredups +# ... or force ignoredups and ignorespace +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# make less more friendly for non-text input files, see lesspipe(1) +#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# prompt +PS1='\[\033[01;34m\]\w\[\033[00m\]$ \[\033[01;32m\]' +trap 'echo -ne "\033[00m"' DEBUG + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + alias dir='dir --color=auto' + alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# some more ls aliases +alias ll='ls -l' +alias la='ls -A' +alias l='ls -CF' + +# enable dvc completion +dvc completion -s bash > /etc/bash_completion.d/dvc + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if [ -f /etc/bash_completion ] && ! shopt -oq posix; then + . /etc/bash_completion +fi + +if [ -f $HOME/start.sh ] ; then + $HOME/start.sh +fi diff --git a/katacoda/generate.bash b/katacoda/generate.bash new file mode 100755 index 0000000..00c8680 --- /dev/null +++ b/katacoda/generate.bash @@ -0,0 +1,25 @@ +#!/bin/bash + + +HERE="$( cd "$(dirname "$0")" ; pwd -P )" + +if [[ -d build ]] ; then + echo "Please delete ${PWD}/build directory first." + exit +fi + +export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" +export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" + +find ${HERE} -name Dockerfile.template | sort | while read -r filepath ; do + SOURCE_DIR=${$(dirname -- "$filepath")#"${HERE}"} + TARGET_DIR=build/${SOURCE_DIR} + if [ ! -d ${TARGET_DIR} ] ; then + mkdir -p ${TARGET_DIR} + fi + cp -fr ${SOURCE_DIR}/* ${TARGET_DIR} + cat ${SOURCE_DIR}/Dockerfile.template | envsubst > ${TARGET_DIR}/Dockerfile + cat ${SOURCE_DIR}/Dockertag.template | envsubst > ${TARGET_DIR}/Dockertag +done + +unset RELEASE_HASH diff --git a/katacoda/get-started/01-initialize/Dockerfile b/katacoda/get-started/01-initialize/Dockerfile.template similarity index 85% rename from katacoda/get-started/01-initialize/Dockerfile rename to katacoda/get-started/01-initialize/Dockerfile.template index 8e7b186..a822e46 100644 --- a/katacoda/get-started/01-initialize/Dockerfile +++ b/katacoda/get-started/01-initialize/Dockerfile.template @@ -1,4 +1,4 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base RUN RELEASE_HASH=${RELEASE_HASH} \ diff --git a/katacoda/get-started/01-initialize/Dockertag b/katacoda/get-started/01-initialize/Dockertag deleted file mode 100644 index 787a355..0000000 --- a/katacoda/get-started/01-initialize/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-initialize diff --git a/katacoda/get-started/01-initialize/Dockertag.template b/katacoda/get-started/01-initialize/Dockertag.template new file mode 100644 index 0000000..628d78c --- /dev/null +++ b/katacoda/get-started/01-initialize/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-initialize diff --git a/katacoda/get-started/02-versioning/Dockerfile b/katacoda/get-started/02-versioning/Dockerfile.template similarity index 88% rename from katacoda/get-started/02-versioning/Dockerfile rename to katacoda/get-started/02-versioning/Dockerfile.template index b2e9abf..ddd627b 100644 --- a/katacoda/get-started/02-versioning/Dockerfile +++ b/katacoda/get-started/02-versioning/Dockerfile.template @@ -1,4 +1,4 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base RUN RELEASE_HASH=${RELEASE_HASH} \ git clone https://github.com/iterative/example-get-started \ diff --git a/katacoda/get-started/02-versioning/Dockertag b/katacoda/get-started/02-versioning/Dockertag deleted file mode 100644 index 9a90c13..0000000 --- a/katacoda/get-started/02-versioning/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-versioning diff --git a/katacoda/get-started/02-versioning/Dockertag.template b/katacoda/get-started/02-versioning/Dockertag.template new file mode 100644 index 0000000..ef2f1e5 --- /dev/null +++ b/katacoda/get-started/02-versioning/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-versioning diff --git a/katacoda/get-started/03-accessing/Dockerfile b/katacoda/get-started/03-accessing/Dockerfile.template similarity index 90% rename from katacoda/get-started/03-accessing/Dockerfile rename to katacoda/get-started/03-accessing/Dockerfile.template index 2258c00..aa07e77 100644 --- a/katacoda/get-started/03-accessing/Dockerfile +++ b/katacoda/get-started/03-accessing/Dockerfile.template @@ -1,6 +1,5 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base -ARG RELEASE_HASH RUN RELEASE_HASH=${RELEASE_HASH} \ apt-get update -y && apt-get install -y \ python3 \ diff --git a/katacoda/get-started/03-accessing/Dockertag b/katacoda/get-started/03-accessing/Dockertag deleted file mode 100644 index 3079d06..0000000 --- a/katacoda/get-started/03-accessing/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-accessing diff --git a/katacoda/get-started/03-accessing/Dockertag.template b/katacoda/get-started/03-accessing/Dockertag.template new file mode 100644 index 0000000..5503121 --- /dev/null +++ b/katacoda/get-started/03-accessing/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-accessing diff --git a/katacoda/get-started/04-stages/Dockerfile b/katacoda/get-started/04-stages/Dockerfile.template similarity index 94% rename from katacoda/get-started/04-stages/Dockerfile rename to katacoda/get-started/04-stages/Dockerfile.template index 26effff..eef14c0 100644 --- a/katacoda/get-started/04-stages/Dockerfile +++ b/katacoda/get-started/04-stages/Dockerfile.template @@ -1,7 +1,5 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base - -ARG RELEASE_HASH RUN RELEASE_HASH=${RELEASE_HASH} \ git clone \ https://github.com/iterative/example-get-started \ diff --git a/katacoda/get-started/04-stages/Dockertag b/katacoda/get-started/04-stages/Dockertag deleted file mode 100644 index a90d4e8..0000000 --- a/katacoda/get-started/04-stages/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-stages diff --git a/katacoda/get-started/04-stages/Dockertag.template b/katacoda/get-started/04-stages/Dockertag.template new file mode 100644 index 0000000..a9b2314 --- /dev/null +++ b/katacoda/get-started/04-stages/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-stages diff --git a/katacoda/get-started/05-params-metrics-plots/Dockerfile b/katacoda/get-started/05-params-metrics-plots/Dockerfile.template similarity index 94% rename from katacoda/get-started/05-params-metrics-plots/Dockerfile rename to katacoda/get-started/05-params-metrics-plots/Dockerfile.template index b2f9bd8..e61c43e 100644 --- a/katacoda/get-started/05-params-metrics-plots/Dockerfile +++ b/katacoda/get-started/05-params-metrics-plots/Dockerfile.template @@ -1,6 +1,5 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base -ARG RELEASE_HASH RUN RELEASE_HASH=${RELEASE_HASH} \ git clone \ https://github.com/iterative/example-get-started \ diff --git a/katacoda/get-started/05-params-metrics-plots/Dockertag b/katacoda/get-started/05-params-metrics-plots/Dockertag deleted file mode 100644 index dd251e2..0000000 --- a/katacoda/get-started/05-params-metrics-plots/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-params diff --git a/katacoda/get-started/05-params-metrics-plots/Dockertag.template b/katacoda/get-started/05-params-metrics-plots/Dockertag.template new file mode 100644 index 0000000..6c4c879 --- /dev/null +++ b/katacoda/get-started/05-params-metrics-plots/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-params diff --git a/katacoda/get-started/06-experiments/Dockerfile b/katacoda/get-started/06-experiments/Dockerfile.template similarity index 84% rename from katacoda/get-started/06-experiments/Dockerfile rename to katacoda/get-started/06-experiments/Dockerfile.template index 1dedc1d..6bd3371 100644 --- a/katacoda/get-started/06-experiments/Dockerfile +++ b/katacoda/get-started/06-experiments/Dockerfile.template @@ -1,6 +1,5 @@ -FROM dvcorg/doc-katacoda:base +FROM ${TAG_PREFIX}/doc-katacoda:base -ARG RELEASE_HASH RUN RELEASE_HASH=${RELEASE_HASH} \ git clone \ https://github.com/iterative/dvc-get-started --branch evaluation \ @@ -19,9 +18,6 @@ RUN RELEASE_HASH=${RELEASE_HASH} \ && pip3 install -r requirements.txt -# RUN RELEASE_HASH=${RELEASE_HASH} \ -# dvc pull - COPY start.sh /root/start.sh EXPOSE 80 diff --git a/katacoda/get-started/06-experiments/Dockertag b/katacoda/get-started/06-experiments/Dockertag deleted file mode 100644 index ca34eda..0000000 --- a/katacoda/get-started/06-experiments/Dockertag +++ /dev/null @@ -1 +0,0 @@ -doc-katacoda:start-experiments diff --git a/katacoda/get-started/06-experiments/Dockertag.template b/katacoda/get-started/06-experiments/Dockertag.template new file mode 100644 index 0000000..53c6d45 --- /dev/null +++ b/katacoda/get-started/06-experiments/Dockertag.template @@ -0,0 +1 @@ +${TAG_PREFIX}/doc-katacoda:start-experiments From 340ede7c4c86262092ecc289352d2768fb83ecc3 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Thu, 13 May 2021 16:52:27 +0000 Subject: [PATCH 06/22] updated Github Action with the script and fixed paths --- katacoda/Dockerfile.template | 35 ---------------------- katacoda/Dockertag.template | 1 - katacoda/bashrc | 57 ------------------------------------ katacoda/generate.bash | 16 ++++++---- 4 files changed, 10 insertions(+), 99 deletions(-) delete mode 100644 katacoda/Dockerfile.template delete mode 100644 katacoda/Dockertag.template delete mode 100644 katacoda/bashrc diff --git a/katacoda/Dockerfile.template b/katacoda/Dockerfile.template deleted file mode 100644 index dd12825..0000000 --- a/katacoda/Dockerfile.template +++ /dev/null @@ -1,35 +0,0 @@ -FROM ubuntu:20.04 - -WORKDIR /root - -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 - -RUN RELEASE_HASH=${RELEASE_HASH} \ - apt-get update -y && apt-get install -y \ - gnupg \ - wget \ - tree - -RUN wget \ - https://dvc.org/deb/dvc.list \ - -O /etc/apt/sources.list.d/dvc.list - -RUN wget -qO - https://dvc.org/deb/iterative.asc | apt-key add - - -RUN RELEASE_HASH=${RELEASE_HASH} \ - apt-get update -y && apt-get install -y \ - dvc - -COPY bashrc .bashrc - -# Configure git user name and email -RUN git config --global user.email "guest@example.com" \ - && git config --global user.name "Guest User" - -CMD ["/bin/bash", "-i"] - - - - - diff --git a/katacoda/Dockertag.template b/katacoda/Dockertag.template deleted file mode 100644 index fea98e1..0000000 --- a/katacoda/Dockertag.template +++ /dev/null @@ -1 +0,0 @@ -${TAG_PREFIX}/doc-katacoda:base diff --git a/katacoda/bashrc b/katacoda/bashrc deleted file mode 100644 index b2939fc..0000000 --- a/katacoda/bashrc +++ /dev/null @@ -1,57 +0,0 @@ -# ~/.bashrc: executed by bash(1) for non-login shells. - -# If not running interactively, don't do anything -[ -z "$PS1" ] && return - -# don't put duplicate lines in the history. See bash(1) for more options -# don't overwrite GNU Midnight Commander's setting of `ignorespace'. -HISTCONTROL=$HISTCONTROL${HISTCONTROL+:}ignoredups -# ... or force ignoredups and ignorespace -HISTCONTROL=ignoreboth - -# append to the history file, don't overwrite it -shopt -s histappend - -# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) - -# check the window size after each command and, if necessary, -# update the values of LINES and COLUMNS. -shopt -s checkwinsize - -# make less more friendly for non-text input files, see lesspipe(1) -#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" - -# prompt -PS1='\[\033[01;34m\]\w\[\033[00m\]$ \[\033[01;32m\]' -trap 'echo -ne "\033[00m"' DEBUG - -# enable color support of ls and also add handy aliases -if [ -x /usr/bin/dircolors ]; then - test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" - alias ls='ls --color=auto' - alias dir='dir --color=auto' - alias vdir='vdir --color=auto' - - alias grep='grep --color=auto' - alias fgrep='fgrep --color=auto' - alias egrep='egrep --color=auto' -fi - -# some more ls aliases -alias ll='ls -l' -alias la='ls -A' -alias l='ls -CF' - -# enable dvc completion -dvc completion -s bash > /etc/bash_completion.d/dvc - -# enable programmable completion features (you don't need to enable -# this, if it's already enabled in /etc/bash.bashrc and /etc/profile -# sources /etc/bash.bashrc). -if [ -f /etc/bash_completion ] && ! shopt -oq posix; then - . /etc/bash_completion -fi - -if [ -f $HOME/start.sh ] ; then - $HOME/start.sh -fi diff --git a/katacoda/generate.bash b/katacoda/generate.bash index 00c8680..964065b 100755 --- a/katacoda/generate.bash +++ b/katacoda/generate.bash @@ -1,9 +1,10 @@ #!/bin/bash +set -vex HERE="$( cd "$(dirname "$0")" ; pwd -P )" -if [[ -d build ]] ; then +if [[ -d ${HERE}/build ]] ; then echo "Please delete ${PWD}/build directory first." exit fi @@ -12,14 +13,17 @@ export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iter export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" find ${HERE} -name Dockerfile.template | sort | while read -r filepath ; do - SOURCE_DIR=${$(dirname -- "$filepath")#"${HERE}"} - TARGET_DIR=build/${SOURCE_DIR} + ABSOLUTE_SOURCE=$(dirname "$filepath") + SOURCE_DIR=$(realpath --relative-to=${HERE} ${ABSOLUTE_SOURCE}) + TARGET_DIR=${HERE}/build/${SOURCE_DIR} if [ ! -d ${TARGET_DIR} ] ; then mkdir -p ${TARGET_DIR} fi - cp -fr ${SOURCE_DIR}/* ${TARGET_DIR} - cat ${SOURCE_DIR}/Dockerfile.template | envsubst > ${TARGET_DIR}/Dockerfile - cat ${SOURCE_DIR}/Dockertag.template | envsubst > ${TARGET_DIR}/Dockertag + cp -fr ${ABSOLUTE_SOURCE}/* ${TARGET_DIR} + cat ${ABSOLUTE_SOURCE}/Dockerfile.template | envsubst > ${TARGET_DIR}/Dockerfile + cat ${ABSOLUTE_SOURCE}/Dockertag.template | envsubst > ${TARGET_DIR}/Dockertag + rm -f ${TARGET_DIR}/Dockerfile.template + rm -f ${TARGET_DIR}/Dockertag.template done unset RELEASE_HASH From 54d42d7373235e2614e3d3db6300157a6532ccd9 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:05:44 +0000 Subject: [PATCH 07/22] removed generate-all call from build --- build-all.bash | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-all.bash b/build-all.bash index c8c68e5..cedae5f 100755 --- a/build-all.bash +++ b/build-all.bash @@ -7,9 +7,6 @@ HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" - -./generate-all.bash - DIR=$(dirname $0) cd "${DIR}" From 179a15de0c867f8bdba90d07a6d127cd89c642ee Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:06:38 +0000 Subject: [PATCH 08/22] fixed zsh indexing in auto tag generation --- build-all.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-all.bash b/build-all.bash index cedae5f..f6cf36a 100755 --- a/build-all.bash +++ b/build-all.bash @@ -16,7 +16,7 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do if [ -f ${tagfile} ] ; then TAG=$(head -n 1 ${tagfile}) else - TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') + TAG=$(echo "${dockerdir}" | tr '/ ' '--') fi echo "BUILDING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" docker build --build-arg RELEASE_HASH=${RELEASE_HASH} -t ${TAG_PREFIX}/${TAG} ${dockerdir}/ From 2399964dd9d5e1a4c53e30bb5ead2705ab9bb76e Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:07:43 +0000 Subject: [PATCH 09/22] checks whether the TAG has TAG_PREFIX as prefix and doesn't add if so --- build-all.bash | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build-all.bash b/build-all.bash index f6cf36a..67dee66 100755 --- a/build-all.bash +++ b/build-all.bash @@ -18,6 +18,12 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do else TAG=$(echo "${dockerdir}" | tr '/ ' '--') fi - echo "BUILDING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" - docker build --build-arg RELEASE_HASH=${RELEASE_HASH} -t ${TAG_PREFIX}/${TAG} ${dockerdir}/ + + if [[ "${TAG}" =~ "${TAG_PREFIX}/" ]] ; then + echo "BUILDING: ${dockerdir} with the tag: ${TAG}" + docker build -t ${TAG} ${dockerdir}/ + else + echo "BUILDING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" + docker build -t ${TAG_PREFIX}/${TAG} ${dockerdir}/ + fi done From 05d8c1acb0756f5122669626149d9315230347bf Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:08:13 +0000 Subject: [PATCH 10/22] fix TAG_PREFIX check --- push-all.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/push-all.bash b/push-all.bash index bda71ab..11def5f 100644 --- a/push-all.bash +++ b/push-all.bash @@ -17,7 +17,7 @@ find $DIR -name Dockerfile | sort | while read -r filepath ; do else TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') fi - if [[ "${TAG}" ~= '.*/.*' ]] ; then + if [[ "${TAG}" =~ "${TAG_PREFIX}/" ]] ; then echo "PUSHING: ${dockerdir} with the tag: ${TAG}" docker push ${TAG} else From 0c5165f4e2057d8f690ade64d90b8337f630bfb3 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:09:22 +0000 Subject: [PATCH 11/22] the workflow runs the generate, build, push separately now --- .github/workflows/build-all.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml index 4a17248..cfd7895 100644 --- a/.github/workflows/build-all.yml +++ b/.github/workflows/build-all.yml @@ -36,12 +36,12 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Generate containers from templates + run: ./generate-all.bash - name: Run build script run: ./build-all.bash - # name: Build and push - # id: docker_build - # uses: docker/build-push-action@v2 - # with: - # push: true - # tags: user/app:latest \ No newline at end of file + - + name: Push the containers + run: ./push-all.bash From 45f9c1101bd38a7f785943da5c12aff6a2692faf Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:14:42 +0000 Subject: [PATCH 12/22] added -u options to all shell scripts --- build-all.bash | 2 +- generate-all.bash | 2 +- katacoda/generate.bash | 2 +- push-all.bash | 2 +- start/generate.bash | 1 + 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build-all.bash b/build-all.bash index 67dee66..ea46116 100755 --- a/build-all.bash +++ b/build-all.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -vex +set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" diff --git a/generate-all.bash b/generate-all.bash index 4e9889a..6b31048 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -vex +set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" diff --git a/katacoda/generate.bash b/katacoda/generate.bash index 964065b..f9fe5a4 100755 --- a/katacoda/generate.bash +++ b/katacoda/generate.bash @@ -1,6 +1,6 @@ #!/bin/bash -set -vex +set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" diff --git a/push-all.bash b/push-all.bash index 11def5f..c907a2b 100644 --- a/push-all.bash +++ b/push-all.bash @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -vex +set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" diff --git a/start/generate.bash b/start/generate.bash index fc815c8..a91d7b4 100755 --- a/start/generate.bash +++ b/start/generate.bash @@ -1,5 +1,6 @@ #!/bin/bash +set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" From 37c2b0eb5e2ee78b0d57e06f615d14b7b9ae44a1 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:17:49 +0000 Subject: [PATCH 13/22] disabled action on pull request --- .github/workflows/build-all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml index cfd7895..ed19e9e 100644 --- a/.github/workflows/build-all.yml +++ b/.github/workflows/build-all.yml @@ -7,8 +7,8 @@ on: # Triggers the workflow on push or pull request events but only for the master branch push: branches: [ master ] - pull_request: - branches: [ master ] + # pull_request: + # branches: [ master ] schedule: - cron: "21 0 * * *" From 32dc461c805fd02c7803d82b0d92c5ea632b1e32 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:25:47 +0000 Subject: [PATCH 14/22] removed RELEASE_HASH from top level scripts --- build-all.bash | 1 - generate-all.bash | 1 - push-all.bash | 1 - 3 files changed, 3 deletions(-) diff --git a/build-all.bash b/build-all.bash index ea46116..8e3b047 100755 --- a/build-all.bash +++ b/build-all.bash @@ -4,7 +4,6 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" -export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" DIR=$(dirname $0) diff --git a/generate-all.bash b/generate-all.bash index 6b31048..9a8b5a7 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -3,7 +3,6 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" -export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" DIR=$(dirname $0) diff --git a/push-all.bash b/push-all.bash index c907a2b..91085c1 100644 --- a/push-all.bash +++ b/push-all.bash @@ -3,7 +3,6 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" -export RELEASE_HASH="${RELEASE_HASH:-$(curl -s https://api.github.com/repos/iterative/dvc/releases/latest | sha256sum | cut -c -8)}" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" DIR=$(dirname $0) From 7f8f9440df307de743ffe4a1da5fcd7463bb1cc2 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 12:38:35 +0000 Subject: [PATCH 15/22] minor fixes on quotations and DIR->HERE --- build-all.bash | 19 ++++++++----------- generate-all.bash | 5 +---- push-all.bash | 19 ++++++++----------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/build-all.bash b/build-all.bash index 8e3b047..2812ad0 100755 --- a/build-all.bash +++ b/build-all.bash @@ -6,23 +6,20 @@ HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -DIR=$(dirname $0) -cd "${DIR}" - -find $DIR -name Dockerfile | sort | while read -r filepath ; do - dockerdir=$(dirname ${filepath}) - tagfile=${dockerdir}/Dockertag - if [ -f ${tagfile} ] ; then - TAG=$(head -n 1 ${tagfile}) +find "${HERE}" -name Dockerfile | sort | while read -r filepath ; do + dockerdir=$(dirname "${filepath}") + tagfile="${dockerdir}/Dockertag" + if [ -f "${tagfile}" ] ; then + TAG=$(head -n 1 "${tagfile}") else - TAG=$(echo "${dockerdir}" | tr '/ ' '--') + TAG=$(echo "${dockerdir:3:100}" | tr '/ ' '--') fi if [[ "${TAG}" =~ "${TAG_PREFIX}/" ]] ; then echo "BUILDING: ${dockerdir} with the tag: ${TAG}" - docker build -t ${TAG} ${dockerdir}/ + docker build -t "${TAG}" "${dockerdir}/" else echo "BUILDING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" - docker build -t ${TAG_PREFIX}/${TAG} ${dockerdir}/ + docker build -t "${TAG_PREFIX}/${TAG}" "${dockerdir}/" fi done diff --git a/generate-all.bash b/generate-all.bash index 9a8b5a7..d35e56d 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -5,10 +5,7 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -DIR=$(dirname $0) -cd "${DIR}" - -find ${DIR} -name generate.* | sort | while read -r filepath ; do +find "${HERE}" -name generate.* | sort | while read -r filepath ; do filename=$(basename -- "$filepath") dirname=$(dirname -- "$filepath") ext="${filename##*.}" diff --git a/push-all.bash b/push-all.bash index 91085c1..25efff4 100644 --- a/push-all.bash +++ b/push-all.bash @@ -5,22 +5,19 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -DIR=$(dirname $0) -cd "${DIR}" - -find $DIR -name Dockerfile | sort | while read -r filepath ; do - dockerdir=$(dirname ${filepath}) - tagfile=${dockerdir}/Dockertag - if [ -f ${tagfile} ] ; then - TAG=$(head -n 1 ${tagfile}) +find "${HERE}" -name Dockerfile | sort | while read -r filepath ; do + dockerdir="$(dirname \"${filepath}\")" + tagfile="${dockerdir}/Dockertag" + if [ -f "${tagfile}" ] ; then + TAG=$(head -n 1 "${tagfile}") else - TAG=$(echo "${dockerdir}[3,100]}" | tr '/ ' '--') + TAG=$(echo "${dockerdir:3:100}" | tr '/ ' '--') fi if [[ "${TAG}" =~ "${TAG_PREFIX}/" ]] ; then echo "PUSHING: ${dockerdir} with the tag: ${TAG}" - docker push ${TAG} + docker push "${TAG}" else echo "PUSHING: ${dockerdir} with the tag: ${TAG_PREFIX}/${TAG}" - docker push ${TAG_PREFIX}/${TAG} + docker push "${TAG_PREFIX}/${TAG}" fi done From db05f23b403be70514b38c3c6a9a3c7be2b89602 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 15:00:01 +0000 Subject: [PATCH 16/22] repository is first cloned to build/ now --- start/generate.bash | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/start/generate.bash b/start/generate.bash index a91d7b4..37cef6b 100755 --- a/start/generate.bash +++ b/start/generate.bash @@ -9,18 +9,20 @@ if [[ -d build ]] ; then exit fi +mkdir build + # Clone dvc-get-started -git clone https://github.com/iterative/dvc-get-started +git clone https://github.com/iterative/dvc-get-started build/dvc-get-started GIT_TAGS=$(git -C dvc-get-started tag --list) for TAG in ${GIT_TAGS} ; do TAG_DIR="build/${TAG}" - mkdir -p ${TAG_DIR} + mkdir -p "${TAG_DIR}" export GIT_TAG=${TAG} - git clone dvc-get-started -b ${GIT_TAG} ${TAG_DIR}/dvc-get-started + git clone build/dvc-get-started -b ${GIT_TAG} ${TAG_DIR}/dvc-get-started cp bashrc ${TAG_DIR}/.bashrc cat Dockerfile.template | envsubst > ${TAG_DIR}/Dockerfile @@ -28,4 +30,3 @@ for TAG in ${GIT_TAGS} ; do done unset GIT_TAG -unset RELEASE_HASH From 2eb442c9634b2c35ca15a49cae8ab5c5548f1d0b Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 15:02:05 +0000 Subject: [PATCH 17/22] cd -> pushd/popd --- generate-all.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate-all.bash b/generate-all.bash index d35e56d..e39d848 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -9,7 +9,7 @@ find "${HERE}" -name generate.* | sort | while read -r filepath ; do filename=$(basename -- "$filepath") dirname=$(dirname -- "$filepath") ext="${filename##*.}" - cd $dirname + pushd $dirname if [[ "$ext" == "bash" ]] ; then bash "$filename" elif [[ "$ext" == "sh" ]] ; then @@ -21,6 +21,6 @@ find "${HERE}" -name generate.* | sort | while read -r filepath ; do else echo "UNSUPPORTED GENERATOR SCRIPT: $filepath" fi - cd - + popd done From b9a12ab03c1ab71d67dc1e3ed8293d26c4bbf0e9 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 16:20:09 +0000 Subject: [PATCH 18/22] quotation marks around paths --- katacoda/generate.bash | 20 ++++++++++---------- start/generate.bash | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/katacoda/generate.bash b/katacoda/generate.bash index f9fe5a4..bdd652e 100755 --- a/katacoda/generate.bash +++ b/katacoda/generate.bash @@ -4,7 +4,7 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" -if [[ -d ${HERE}/build ]] ; then +if [[ -d "${HERE}/build" ]] ; then echo "Please delete ${PWD}/build directory first." exit fi @@ -14,16 +14,16 @@ export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" find ${HERE} -name Dockerfile.template | sort | while read -r filepath ; do ABSOLUTE_SOURCE=$(dirname "$filepath") - SOURCE_DIR=$(realpath --relative-to=${HERE} ${ABSOLUTE_SOURCE}) - TARGET_DIR=${HERE}/build/${SOURCE_DIR} - if [ ! -d ${TARGET_DIR} ] ; then - mkdir -p ${TARGET_DIR} + SOURCE_DIR=$(realpath --relative-to="${HERE}" "${ABSOLUTE_SOURCE}") + TARGET_DIR="${HERE}/build/${SOURCE_DIR}" + if [ ! -d "${TARGET_DIR}" ] ; then + mkdir -p "${TARGET_DIR}" fi - cp -fr ${ABSOLUTE_SOURCE}/* ${TARGET_DIR} - cat ${ABSOLUTE_SOURCE}/Dockerfile.template | envsubst > ${TARGET_DIR}/Dockerfile - cat ${ABSOLUTE_SOURCE}/Dockertag.template | envsubst > ${TARGET_DIR}/Dockertag - rm -f ${TARGET_DIR}/Dockerfile.template - rm -f ${TARGET_DIR}/Dockertag.template + cp -fr ${ABSOLUTE_SOURCE}/* "${TARGET_DIR}" + cat "${ABSOLUTE_SOURCE}/Dockerfile.template" | envsubst > "${TARGET_DIR}/Dockerfile" + cat "${ABSOLUTE_SOURCE}/Dockertag.template" | envsubst > "${TARGET_DIR}/Dockertag" + rm -f "${TARGET_DIR}/Dockerfile.template" + rm -f "${TARGET_DIR}/Dockertag.template" done unset RELEASE_HASH diff --git a/start/generate.bash b/start/generate.bash index 37cef6b..9195764 100755 --- a/start/generate.bash +++ b/start/generate.bash @@ -20,13 +20,13 @@ for TAG in ${GIT_TAGS} ; do TAG_DIR="build/${TAG}" mkdir -p "${TAG_DIR}" - export GIT_TAG=${TAG} + export GIT_TAG="${TAG}" - git clone build/dvc-get-started -b ${GIT_TAG} ${TAG_DIR}/dvc-get-started + git clone build/dvc-get-started -b "${GIT_TAG}" "${TAG_DIR}/dvc-get-started" - cp bashrc ${TAG_DIR}/.bashrc - cat Dockerfile.template | envsubst > ${TAG_DIR}/Dockerfile - cat Dockertag.template | envsubst > ${TAG_DIR}/Dockertag + cp bashrc "${TAG_DIR}/.bashrc" + cat Dockerfile.template | envsubst > "${TAG_DIR}/Dockerfile" + cat Dockertag.template | envsubst > "${TAG_DIR}/Dockertag" done unset GIT_TAG From 7dff8378f0880b2f79a7c170569ff6af582b5776 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 16:20:34 +0000 Subject: [PATCH 19/22] find $HERE -> find . --- build-all.bash | 2 +- generate-all.bash | 2 +- push-all.bash | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 push-all.bash diff --git a/build-all.bash b/build-all.bash index 2812ad0..442818a 100755 --- a/build-all.bash +++ b/build-all.bash @@ -6,7 +6,7 @@ HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -find "${HERE}" -name Dockerfile | sort | while read -r filepath ; do +find . -name Dockerfile | sort | while read -r filepath ; do dockerdir=$(dirname "${filepath}") tagfile="${dockerdir}/Dockertag" if [ -f "${tagfile}" ] ; then diff --git a/generate-all.bash b/generate-all.bash index e39d848..a97eaa3 100755 --- a/generate-all.bash +++ b/generate-all.bash @@ -5,7 +5,7 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -find "${HERE}" -name generate.* | sort | while read -r filepath ; do +find . -name generate.* | sort | while read -r filepath ; do filename=$(basename -- "$filepath") dirname=$(dirname -- "$filepath") ext="${filename##*.}" diff --git a/push-all.bash b/push-all.bash old mode 100644 new mode 100755 index 25efff4..1ac57b4 --- a/push-all.bash +++ b/push-all.bash @@ -5,8 +5,8 @@ set -uvex HERE="$( cd "$(dirname "$0")" ; pwd -P )" export TAG_PREFIX="${TAG_PREFIX:-dvcorg}" -find "${HERE}" -name Dockerfile | sort | while read -r filepath ; do - dockerdir="$(dirname \"${filepath}\")" +find . -name Dockerfile | sort | while read -r filepath ; do + dockerdir=$(dirname "${filepath}") tagfile="${dockerdir}/Dockertag" if [ -f "${tagfile}" ] ; then TAG=$(head -n 1 "${tagfile}") From 2676eef6c588f87ffb0b139ff2ce3f7a13a3914b Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 16:20:55 +0000 Subject: [PATCH 20/22] add TAG_PREFIX to Dockertag --- start/Dockertag.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start/Dockertag.template b/start/Dockertag.template index 0255d2a..1826126 100644 --- a/start/Dockertag.template +++ b/start/Dockertag.template @@ -1 +1 @@ -dvcorg/doc-start:${GIT_TAG} +${TAG_PREFIX}/doc-start:${GIT_TAG} From b2ee0d50dc887d4e8b042aee77f9cb2412c60602 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 16:21:12 +0000 Subject: [PATCH 21/22] run dvc pull --- start/Dockerfile.template | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/start/Dockerfile.template b/start/Dockerfile.template index f7f0af5..854e08a 100644 --- a/start/Dockerfile.template +++ b/start/Dockerfile.template @@ -17,6 +17,10 @@ RUN RELEASE_HASH=${RELEASE_HASH} \ COPY dvc-get-started dvc-get-started COPY .bashrc .bashrc +WORKDIR /root/dvc-get-started + +RUN bash -c "if [ -d .dvc ] ; then dvc pull ; fi" + # Configure git user name and email RUN git config --global user.email "guest@example.com" \ && git config --global user.name "Guest User" From dc446860465d638584105baec9c498d23a5eeb79 Mon Sep 17 00:00:00 2001 From: "github@emresult.com" Date: Fri, 14 May 2021 16:21:23 +0000 Subject: [PATCH 22/22] for -> while --- start/generate.bash | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/start/generate.bash b/start/generate.bash index 9195764..fbc9b8b 100755 --- a/start/generate.bash +++ b/start/generate.bash @@ -14,9 +14,7 @@ mkdir build # Clone dvc-get-started git clone https://github.com/iterative/dvc-get-started build/dvc-get-started -GIT_TAGS=$(git -C dvc-get-started tag --list) - -for TAG in ${GIT_TAGS} ; do +git -C build/dvc-get-started tag --list | while read -r TAG ; do TAG_DIR="build/${TAG}" mkdir -p "${TAG_DIR}"