diff --git a/scripts/build_package.sh b/scripts/build_package.sh index 7c61b32cf3..4818d7a6e2 100755 --- a/scripts/build_package.sh +++ b/scripts/build_package.sh @@ -4,8 +4,8 @@ set -e set -x if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" >&2 - exit 1 + echo "Please run this script from repository root" >&2 + exit 1 fi echo 'PKG = "pip"' >dvc/utils/build.py @@ -15,15 +15,15 @@ python setup.py bdist_wheel --universal # Make sure we have a correct version if [[ -n "$TRAVIS_TAG" ]]; then - pip uninstall -y dvc - if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 - exit 1 - fi - pip install dist/dvc-*.whl - if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then - echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 - exit 1 - fi - pip uninstall -y dvc + pip uninstall -y dvc + if which dvc; then + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 + fi + pip install dist/dvc-*.whl + if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then + echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 + exit 1 + fi + pip uninstall -y dvc fi diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index a43f1c4ab8..83386e40d7 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -4,13 +4,13 @@ set -e set -x if [[ "$(uname)" == 'Linux' ]]; then - INSTALL_DIR=usr - FPM_FLAGS= + INSTALL_DIR=usr + FPM_FLAGS= else - INSTALL_DIR=usr/local - FPM_FLAGS='--osxpkg-identifier-prefix com.iterative' - FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh' - FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh' + INSTALL_DIR=usr/local + FPM_FLAGS='--osxpkg-identifier-prefix com.iterative' + FPM_FLAGS+=' --after-install scripts/fpm/after-install.sh' + FPM_FLAGS+=' --after-remove scripts/fpm/after-remove.sh' fi BUILD_DIR=build @@ -21,121 +21,121 @@ LIB_DIR=$BUILD_DIR/$INSTALL_DIR/lib FPM_PACKAGE_DIRS="usr" ZSH_CMPLT_DIR=usr/share/zsh/site-functions/_dvc if [[ "$(uname)" == 'Linux' ]]; then - BASH_CMPLT_DIR=etc/bash_completion.d - FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc" + BASH_CMPLT_DIR=etc/bash_completion.d + FPM_PACKAGE_DIRS="$FPM_PACKAGE_DIRS etc" else - BASH_CMPLT_DIR=usr/local/etc/bash_completion.d + BASH_CMPLT_DIR=usr/local/etc/bash_completion.d fi print_error() { - echo -e "\e[31m$1\e[0m" >&2 + echo -e "\e[31m$1\e[0m" >&2 } if [ ! -d "dvc" ]; then - print_error "Please run this script from repository root" - exit 1 + print_error "Please run this script from repository root" + exit 1 fi trap 'print_error "FAIL"; exit 1' ERR print_info() { - echo -e "\e[32m$1\e[0m" + echo -e "\e[32m$1\e[0m" } command_exists() { - command -v $1 >/dev/null 2>&1 + command -v $1 >/dev/null 2>&1 } fpm_build() { - print_info "Building $1..." - VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))") - fpm -s dir \ - -f \ - -t $1 \ - --description "$DESC" \ - $FPM_FLAGS \ - -n dvc \ - -v $VERSION \ - -C $BUILD_DIR \ - $FPM_PACKAGE_DIRS + print_info "Building $1..." + VERSION=$(python -c "import dvc; from dvc import __version__; print(str(__version__))") + fpm -s dir \ + -f \ + -t $1 \ + --description "$DESC" \ + $FPM_FLAGS \ + -n dvc \ + -v $VERSION \ + -C $BUILD_DIR \ + $FPM_PACKAGE_DIRS } cleanup() { - print_info "Cleaning up..." - rm -rf build + print_info "Cleaning up..." + rm -rf build } install_dependencies() { - print_info "Installing fpm..." - if command_exists dnf; then - sudo dnf install ruby-devel gcc make rpm-build - elif command_exists yum; then - sudo yum install ruby-devel gcc make rpm-build - elif command_exists apt-get; then - sudo apt-get update -y - sudo apt-get install ruby-dev build-essential rpm python-pip python-dev - elif command_exists brew; then - brew install ruby - else - echo "Unable to install fpm dependencies" && exit 1 - fi - - gem install --no-document fpm - - # https://github.com/iterative/dvc/issues/2970 - gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler - - print_info "Installing requirements..." - pip install .[all] - pip install -r scripts/build-requirements.txt + print_info "Installing fpm..." + if command_exists dnf; then + sudo dnf install ruby-devel gcc make rpm-build + elif command_exists yum; then + sudo yum install ruby-devel gcc make rpm-build + elif command_exists apt-get; then + sudo apt-get update -y + sudo apt-get install ruby-dev build-essential rpm python-pip python-dev + elif command_exists brew; then + brew install ruby + else + echo "Unable to install fpm dependencies" && exit 1 + fi + + gem install --no-document fpm + + # https://github.com/iterative/dvc/issues/2970 + gem uninstall -i /Users/travis/.rvm/gems/ruby-2.4.3@global rubygems-bundler + + print_info "Installing requirements..." + pip install .[all] + pip install -r scripts/build-requirements.txt } build_dvc() { - print_info "Building dvc binary..." - pyinstaller \ - --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \ - --name dvc \ - --distpath $LIB_DIR \ - --specpath $BUILD_DIR - - $LIB_DIR/dvc/dvc --help - - # NOTE: in osxpkg fpm replaces symlinks with actual file that it - # points to, so we need to use after-install hook. See FPM_FLAGS - # above. - if [[ "$(uname)" == 'Linux' ]]; then - mkdir -p $BIN_DIR - pushd $BIN_DIR - ln -s ../lib/dvc/dvc dvc - popd - $BIN_DIR/dvc --help - fi - - # NOTE: temporarily not adding scripts to mac package. See [1] - # [1] https://github.com/iterative/dvc/issues/2585 - if [[ "$(uname)" == 'Linux' ]]; then - mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR - cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc - - mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR - cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR - fi + print_info "Building dvc binary..." + pyinstaller \ + --additional-hooks-dir $(pwd)/scripts/hooks dvc/__main__.py \ + --name dvc \ + --distpath $LIB_DIR \ + --specpath $BUILD_DIR + + $LIB_DIR/dvc/dvc --help + + # NOTE: in osxpkg fpm replaces symlinks with actual file that it + # points to, so we need to use after-install hook. See FPM_FLAGS + # above. + if [[ "$(uname)" == 'Linux' ]]; then + mkdir -p $BIN_DIR + pushd $BIN_DIR + ln -s ../lib/dvc/dvc dvc + popd + $BIN_DIR/dvc --help + fi + + # NOTE: temporarily not adding scripts to mac package. See [1] + # [1] https://github.com/iterative/dvc/issues/2585 + if [[ "$(uname)" == 'Linux' ]]; then + mkdir -p $BUILD_DIR/$BASH_CMPLT_DIR + cp scripts/completion/dvc.bash $BUILD_DIR/$BASH_CMPLT_DIR/dvc + + mkdir -p $BUILD_DIR/$ZSH_CMPLT_DIR + cp scripts/completion/dvc.zsh $BUILD_DIR/$ZSH_CMPLT_DIR + fi } build() { - cleanup - echo "PKG = \"$1\"" >dvc/utils/build.py - build_dvc - fpm_build $1 + cleanup + echo "PKG = \"$1\"" >dvc/utils/build.py + build_dvc + fpm_build $1 } install_dependencies if [[ "$(uname)" == 'Linux' ]]; then - build rpm - build deb + build rpm + build deb else - build osxpkg + build osxpkg fi cleanup diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh index 1ef6776bd3..93e7d51cc5 100755 --- a/scripts/build_snap.sh +++ b/scripts/build_snap.sh @@ -3,24 +3,24 @@ set -e set -x if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" - exit 1 + echo "Please run this script from repository root" + exit 1 fi sudo snapcraft --use-lxd pip uninstall -y dvc if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 - exit 1 + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 fi sudo snap install --dangerous --classic dvc_*.snap if [[ -n "$TRAVIS_TAG" ]]; then - # Make sure we have a correct version - if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then - echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 - exit 1 - fi + # Make sure we have a correct version + if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then + echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 + exit 1 + fi fi # ensure basic commands can run # N.B.: cannot run `dvc get` on travis (#2956) diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index c154a62b6a..451179c9bb 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -6,53 +6,53 @@ set -e scriptdir="$(dirname $0)" if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - ulimit -a - sudo sysctl -w kern.maxproc=2048 - sudo sysctl -w kern.maxprocperuid=2048 - echo '\nulimit -u 2048' >>~/.bash_profile - ulimit -a + ulimit -a + sudo sysctl -w kern.maxproc=2048 + sudo sysctl -w kern.maxprocperuid=2048 + echo '\nulimit -u 2048' >>~/.bash_profile + ulimit -a fi echo >env.sh if [[ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]]; then - if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then - # NOTE: ssh keys for ssh test to be able to ssh to the localhost - ssh-keygen -t rsa -N "" -f mykey - mkdir -p ~/.ssh - cp mykey ~/.ssh/id_rsa - cp mykey.pub ~/.ssh/id_rsa.pub - cat mykey.pub >>~/.ssh/authorized_keys - ssh-keyscan localhost >>~/.ssh/known_hosts - ssh localhost ls &>/dev/null - ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts - ssh 127.0.0.1 ls &>/dev/null - ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts - ssh 0.0.0.0 ls &>/dev/null - fi - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then - bash "$scriptdir/install_azurite.sh" - bash "$scriptdir/install_oss.sh" - bash "$scriptdir/install_hadoop.sh" - fi - - if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then - brew install openssl - $scriptdir/retry.sh brew cask install google-cloud-sdk - fi + if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + # NOTE: ssh keys for ssh test to be able to ssh to the localhost + ssh-keygen -t rsa -N "" -f mykey + mkdir -p ~/.ssh + cp mykey ~/.ssh/id_rsa + cp mykey.pub ~/.ssh/id_rsa.pub + cat mykey.pub >>~/.ssh/authorized_keys + ssh-keyscan localhost >>~/.ssh/known_hosts + ssh localhost ls &>/dev/null + ssh-keyscan 127.0.0.1 >>~/.ssh/known_hosts + ssh 127.0.0.1 ls &>/dev/null + ssh-keyscan 0.0.0.0 >>~/.ssh/known_hosts + ssh 0.0.0.0 ls &>/dev/null + fi + + if [ "$TRAVIS_OS_NAME" == "linux" ]; then + bash "$scriptdir/install_azurite.sh" + bash "$scriptdir/install_oss.sh" + bash "$scriptdir/install_hadoop.sh" + fi + + if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then + brew install openssl + $scriptdir/retry.sh brew cask install google-cloud-sdk + fi fi if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - $scriptdir/retry.sh choco install python --version 3.7.5 - echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh + $scriptdir/retry.sh choco install python --version 3.7.5 + echo 'PATH="/c/Python37:/c/Python37/Scripts:$PATH"' >>env.sh elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - ln -s -f /usr/local/bin/python3 /usr/local/bin/python - ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip + ln -s -f /usr/local/bin/python3 /usr/local/bin/python + ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip fi if [[ -n "$TRAVIS_TAG" ]]; then - echo "export SNAP_CHANNEL=stable" >>env.sh + echo "export SNAP_CHANNEL=stable" >>env.sh else - echo "export SNAP_CHANNEL=edge" >>env.sh + echo "export SNAP_CHANNEL=edge" >>env.sh fi diff --git a/scripts/ci/check_python.sh b/scripts/ci/check_python.sh index 77e86ecbef..b4ca089329 100644 --- a/scripts/ci/check_python.sh +++ b/scripts/ci/check_python.sh @@ -4,15 +4,15 @@ set -e set -x if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then - where python - where pip + where python + where pip else - which python - which pip + which python + which pip fi py_ver="$(python -c 'import sys; print(sys.version[0])')" if [[ "$py_ver" != 3 ]]; then - exit 1 + exit 1 fi diff --git a/scripts/ci/deploy_condition.sh b/scripts/ci/deploy_condition.sh index 1309f76289..6b95494347 100755 --- a/scripts/ci/deploy_condition.sh +++ b/scripts/ci/deploy_condition.sh @@ -3,13 +3,13 @@ set -e if [[ $TRAVIS_EVENT_TYPE = pull_request || $TRAVIS_EVENT_TPYE = cron ]]; then - exit 2 + exit 2 fi # positional args are assumed to be file glob patterns to deploy if [[ ${#} -ge 1 ]]; then - # ensure at least one file exists - [[ -n "$(ls $@ 2>/dev/null)" ]] || exit 3 + # ensure at least one file exists + [[ -n "$(ls $@ 2>/dev/null)" ]] || exit 3 fi echo true diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh index d91c207be5..425b6eaf62 100644 --- a/scripts/ci/install.sh +++ b/scripts/ci/install.sh @@ -14,9 +14,9 @@ git config --global user.email "dvctester@example.com" git config --global user.name "DVC Tester" if [[ "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set region us-east-2 + aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure set region us-east-2 - openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json + openssl enc -d -aes-256-cbc -md md5 -k $GCP_CREDS -in scripts/ci/gcp-creds.json.enc -out scripts/ci/gcp-creds.json fi diff --git a/scripts/ci/install_azurite.sh b/scripts/ci/install_azurite.sh index 30a7ef9437..d02223d8a0 100755 --- a/scripts/ci/install_azurite.sh +++ b/scripts/ci/install_azurite.sh @@ -12,9 +12,9 @@ sudo apt-get install -y docker-ce # run azurite sudo docker run -d --restart always -e executable=blob -p 10000:10000 \ - --tmpfs /opt/azurite/folder \ - mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \ - azurite -l /data --blobHost 0.0.0.0 + --tmpfs /opt/azurite/folder \ + mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \ + azurite -l /data --blobHost 0.0.0.0 # save secrets echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh diff --git a/scripts/ci/retry.sh b/scripts/ci/retry.sh index cd2ff129fa..07458c9948 100755 --- a/scripts/ci/retry.sh +++ b/scripts/ci/retry.sh @@ -4,8 +4,8 @@ set -e N_RETRIES=3 for i in $(seq $N_RETRIES); do - echo "Attempt #$i" - $@ && exit $? + echo "Attempt #$i" + $@ && exit $? done exit $? diff --git a/scripts/ci/script.sh b/scripts/ci/script.sh index f9b6a72477..2ad7ba26c4 100644 --- a/scripts/ci/script.sh +++ b/scripts/ci/script.sh @@ -6,7 +6,7 @@ set -e python -mtests if [[ "$TRAVIS_PULL_REQUEST" == "false" && \ - "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then - pip install codecov - codecov + "$TRAVIS_SECURE_ENV_VARS" == "true" ]]; then + pip install codecov + codecov fi diff --git a/scripts/completion/dvc.bash b/scripts/completion/dvc.bash index 40a2604410..6505c1398e 100644 --- a/scripts/completion/dvc.bash +++ b/scripts/completion/dvc.bash @@ -72,30 +72,30 @@ _dvc_version='' # ${!x} -> ${hello} -> "world" # _dvc() { - replace_hyphen() { - echo $(echo $1 | sed 's/-/_/g') - } + replace_hyphen() { + echo $(echo $1 | sed 's/-/_/g') + } - local word="${COMP_WORDS[COMP_CWORD]}" + local word="${COMP_WORDS[COMP_CWORD]}" - COMPREPLY=() + COMPREPLY=() - if [ "${COMP_CWORD}" -eq 1 ]; then - case "$word" in - -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;; - *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;; - esac - elif [ "${COMP_CWORD}" -eq 2 ]; then - local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})" + if [ "${COMP_CWORD}" -eq 1 ]; then + case "$word" in + -*) COMPREPLY=($(compgen -W "$_dvc_options" -- "$word")) ;; + *) COMPREPLY=($(compgen -W "$_dvc_commands" -- "$word")) ;; + esac + elif [ "${COMP_CWORD}" -eq 2 ]; then + local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})" - COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word")) - elif [ "${COMP_CWORD}" -eq 3 ]; then - local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})" + COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word")) + elif [ "${COMP_CWORD}" -eq 3 ]; then + local options_list="_dvc_$(replace_hyphen ${COMP_WORDS[1]})_$(replace_hyphen ${COMP_WORDS[2]})" - COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word")) - fi + COMPREPLY=($(compgen -W "$_dvc_global_options ${!options_list}" -- "$word")) + fi - return 0 + return 0 } complete -F _dvc dvc diff --git a/scripts/pypi_upload.sh b/scripts/pypi_upload.sh index 572bc0a2ee..5c9115662d 100755 --- a/scripts/pypi_upload.sh +++ b/scripts/pypi_upload.sh @@ -3,8 +3,8 @@ set -e if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" - exit 1 + echo "Please run this script from repository root" + exit 1 fi rm -rf dist/