diff --git a/.dockerignore b/.dockerignore index 1dab395bfc1..bf1bef0cc55 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,10 @@ * # Make an exception for azure-cli source code !src/* +# Make an exception for private SDKs if they exist +!privates/* +# Make an exception for tab completion script +!az.completion # Make an exception for the license file !LICENSE.txt # Exclude build droppings, as mentioned in .gitignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b00a5e7b7e..cb23c302699 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,6 @@ /tools/ @troydai @derekbekoe /scripts/ @troydai @derekbekoe -/packaged_releases/ @derekbekoe /src/azure-cli-testsdk/ @troydai /src/command_modules/azure-cli-acr/ @djyou /src/command_modules/azure-cli-acs/ @rjtsdl diff --git a/.gitignore b/.gitignore index 82bbdfbf0df..db140dc8ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ env27/ obj/ dist/ MANIFEST -packaged_releases/windows/out +build_scripts/windows/out # Result of running python setup.py install/pip install -e RECORD.txt diff --git a/Dockerfile b/Dockerfile index d65262077e1..fb5c6d85332 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,24 @@ # Licensed under the MIT License. See License.txt in the project root for license information. #--------------------------------------------------------------------------------------------- -# This Dockerfile uses the latest code from the Git repo. -# Clone the repo then run 'docker build' with this Dockerfile file to get the latest versions of -# *all* CLI modules as in the Git repo. - FROM python:3.5.2-alpine +ARG CLI_VERSION + +# Metadata as defined at http://label-schema.org +ARG BUILD_DATE +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.vendor="Microsoft" \ + org.label-schema.name="Azure CLI 2.0" \ + org.label-schema.version=$CLI_VERSION \ + org.label-schema.license="MIT" \ + org.label-schema.description="The Azure CLI 2.0 is the new Azure CLI and is applicable when you use the Resource Manager deployment model." \ + org.label-schema.url="https://docs.microsoft.com/en-us/cli/azure/overview" \ + org.label-schema.usage="https://docs.microsoft.com/en-us/cli/azure/install-az-cli2#docker" \ + org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.vcs-url="https://github.com/Azure/azure-cli.git" \ + org.label-schema.docker.cmd="docker run -v ${HOME}:/root -it azuresdk/azure-cli-python:" + WORKDIR azure-cli COPY . /azure-cli # pip wheel - required for CLI packaging @@ -29,26 +41,15 @@ RUN wget https://github.com/jmespath/jp/releases/download/0.1.2/jp-linux-amd64 - # 3. Temporary fix - install azure-nspkg to remove import of pkg_resources in azure/__init__.py (to improve performance) RUN /bin/bash -c 'TMP_PKG_DIR=$(mktemp -d); \ for d in src/azure-cli src/azure-cli-core src/azure-cli-nspkg src/azure-cli-command_modules-nspkg src/command_modules/azure-cli-*/; \ - do cd $d; python setup.py bdist_wheel -d $TMP_PKG_DIR; cd -; \ - done; \ - MODULE_NAMES=""; \ - for f in $TMP_PKG_DIR/*; \ - do MODULE_NAMES="$MODULE_NAMES $f"; \ + do cd $d; echo $d; python setup.py bdist_wheel -d $TMP_PKG_DIR; cd -; \ done; \ - pip install --no-cache-dir $MODULE_NAMES; \ + [ -d privates ] && cp privates/*.whl $TMP_PKG_DIR; \ + all_modules=`find $TMP_PKG_DIR -name "*.whl"`; \ + pip install --no-cache-dir $all_modules; \ pip install --no-cache-dir --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg;' # Tab completion -RUN echo -e "\ -_python_argcomplete() {\n\ - local IFS='\v'\n\ - COMPREPLY=( \$(IFS=\"\$IFS\" COMP_LINE=\"\$COMP_LINE\" COMP_POINT=\"\$COMP_POINT\" _ARGCOMPLETE_COMP_WORDBREAKS=\"\$COMP_WORDBREAKS\" _ARGCOMPLETE=1 \"\$1\" 8>&1 9>&2 1>/dev/null 2>/dev/null) )\n\ - if [[ \$? != 0 ]]; then\n\ - unset COMPREPLY\n\ - fi\n\ -}\n\ -complete -o nospace -F _python_argcomplete \"az\"\n\ -" > ~/.bashrc +RUN cat /azure-cli/az.completion > ~/.bashrc WORKDIR / diff --git a/packaged_releases/az.completion b/az.completion similarity index 100% rename from packaged_releases/az.completion rename to az.completion diff --git a/packaged_releases/debian/README.md b/build_scripts/debian/README.md similarity index 50% rename from packaged_releases/debian/README.md rename to build_scripts/debian/README.md index 5c6917cac82..f1c92138bdb 100644 --- a/packaged_releases/debian/README.md +++ b/build_scripts/debian/README.md @@ -7,22 +7,16 @@ Updating the Debian package On a build machine (e.g. new Ubuntu 14.04 VM), run the build script. For example: - -First copy the build scripts onto the build machine. -``` -> ~/debian_build.sh; editor ~/debian_build.sh -> ~/debian_dir_creator.sh; editor ~/debian_dir_creator.sh -chmod +x ~/debian_build.sh ~/debian_dir_creator.sh -``` - -Then execute it with the appropriate environment variable values. ``` +git clone https://github.com/azure/azure-cli +cd azure-cli export CLI_VERSION=2.0.9 \ - && export CLI_DOWNLOAD_SHA256=e74150b2db2975e8b17710eb7ef270ded16e6a8c27f77929544533f6b4c33b76 \ && export BUILD_ARTIFACT_DIR=$(mktemp -d)\ - && ~/debian_build.sh ~/debian_dir_creator.sh + && build_scripts/debian/build.sh $(pwd) ``` +Note: The paths above have to be full paths, not relative otherwise the build will fail. + Now you have built the package, upload the package to the apt repository. diff --git a/packaged_releases/debian/debian_build.sh b/build_scripts/debian/build.sh similarity index 67% rename from packaged_releases/debian/debian_build.sh rename to build_scripts/debian/build.sh index be072977e72..6f2d05283ce 100755 --- a/packaged_releases/debian/debian_build.sh +++ b/build_scripts/debian/build.sh @@ -11,19 +11,16 @@ set -ex if [ -z "$1" ] then - echo "First argument should be path to executable debian directory creator." + echo "Argument should be path to local repo." exit 1 fi -local_repo=$2 -if [ -z "$local_repo" ] - then - : "${CLI_DOWNLOAD_SHA256:?CLI_DOWNLOAD_SHA256 environment variable not set.}" -fi +local_repo=$1 sudo apt-get update -debian_directory_creator=$1 +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +debian_directory_creator=$script_dir/dir_creator.sh # Install dependencies for the build sudo apt-get install -y libssl-dev libffi-dev python3-dev debhelper @@ -31,30 +28,15 @@ sudo apt-get install -y libssl-dev libffi-dev python3-dev debhelper tmp_pkg_dir=$(mktemp -d) working_dir=$(mktemp -d) cd $working_dir -if [ -z "$local_repo" ] +source_dir=$local_repo +deb_file=$local_repo/../azure-cli_${CLI_VERSION}-1_all.deb +az_completion_file=$source_dir/az.completion +# clean up old build output +if [ -d "$source_dir/debian" ] then - source_archive=$working_dir/azure-cli-${CLI_VERSION}.tar.gz - source_dir=$working_dir/azure-cli-${CLI_VERSION} - deb_file=$working_dir/azure-cli_${CLI_VERSION}-1_all.deb - az_completion_file=$source_dir/az.completion - wget https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_${CLI_VERSION}.tar.gz -qO $source_archive - echo "$CLI_DOWNLOAD_SHA256 $source_archive" | sha256sum -c - - mkdir $source_dir - # Extract archive - archive_extract_dir=$(mktemp -d) - tar -xvzf $source_archive -C $archive_extract_dir - cp -r $archive_extract_dir/azure-cli_packaged_${CLI_VERSION}/* $source_dir - else - source_dir=$local_repo - deb_file=$local_repo/../azure-cli_${CLI_VERSION}-1_all.deb - az_completion_file=$source_dir/packaged_releases/az.completion - # clean up old build output - if [ -d "$source_dir/debian" ] - then - rm -rf $source_dir/debian - fi - cp $local_repo/privates/*.whl $tmp_pkg_dir + rm -rf $source_dir/debian fi +[ -d $local_repo/privates ] && cp $local_repo/privates/*.whl $tmp_pkg_dir # Build Python from source and include python_dir=$(mktemp -d) @@ -74,7 +56,8 @@ make install # It does not affect the built .deb file though. $source_dir/python_env/bin/pip3 install wheel for d in $source_dir/src/azure-cli $source_dir/src/azure-cli-core $source_dir/src/azure-cli-nspkg $source_dir/src/azure-cli-command_modules-nspkg $source_dir/src/command_modules/azure-cli-*/; do cd $d; $source_dir/python_env/bin/python3 setup.py bdist_wheel -d $tmp_pkg_dir; cd -; done; -$source_dir/python_env/bin/pip3 install azure-cli --find-links $tmp_pkg_dir +all_modules=`find $tmp_pkg_dir -name "*.whl"` +$source_dir/python_env/bin/pip3 install $all_modules $source_dir/python_env/bin/pip3 install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg # WORKAROUND: Newer versions of cryptography do not work on Bash on Windows / WSL - see https://github.com/Azure/azure-cli/issues/4154 # If you *have* to use a newer version of cryptography in the future, verify that it works on WSL also. diff --git a/packaged_releases/debian/debian_dir_creator.sh b/build_scripts/debian/dir_creator.sh similarity index 100% rename from packaged_releases/debian/debian_dir_creator.sh rename to build_scripts/debian/dir_creator.sh diff --git a/packaged_releases/docker/README.md b/build_scripts/docker/README.md similarity index 80% rename from packaged_releases/docker/README.md rename to build_scripts/docker/README.md index aeb0a8bcbd4..a32b9e7e08f 100644 --- a/packaged_releases/docker/README.md +++ b/build_scripts/docker/README.md @@ -5,9 +5,9 @@ The Docker image is available at https://hub.docker.com/r/azuresdk/azure-cli-pyt Updating the Docker image ------------------------- -1. Run `docker build` with this Dockerfile. +1. Run `docker build` with the Dockerfile. When tagging this Docker image, choose an appropriate version number. - e.g.: ``sudo docker build --no-cache --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" --build-arg CLI_VERSION=${CLI_VERSION} --build-arg CLI_DOWNLOAD_SHA256=${CLI_DOWNLOAD_SHA256} -f Dockerfile -t azuresdk/azure-cli-python:${CLI_VERSION} .`` + e.g.: ``sudo docker build --no-cache --build-arg BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`" --build-arg CLI_VERSION=${CLI_VERSION} -f Dockerfile -t azuresdk/azure-cli-python:${CLI_VERSION} .`` 2. Push the image to the registry, e.g.: `sudo docker push azuresdk/azure-cli-python:${CLI_VERSION}` 3. Create a PR to commit the Dockerfile changes back to the repository. diff --git a/packaged_releases/homebrew/README.md b/build_scripts/homebrew/README.md similarity index 100% rename from packaged_releases/homebrew/README.md rename to build_scripts/homebrew/README.md diff --git a/packaged_releases/rpm/README.md b/build_scripts/rpm/README.md similarity index 53% rename from packaged_releases/rpm/README.md rename to build_scripts/rpm/README.md index 506324b22f6..4dc373f35d9 100644 --- a/packaged_releases/rpm/README.md +++ b/build_scripts/rpm/README.md @@ -7,31 +7,18 @@ Building the RPM package On a build machine (e.g. new CentOS 7 VM) run the following. Install dependencies required to build: +Required for rpm build tools & required to build the CLI. ``` -# Required for rpm build tools. -sudo yum install -y gcc rpm-build rpm-devel rpmlint make python bash coreutils diffutils patch rpmdevtools - -# Required to build the CLI. -sudo yum install -y gcc python libffi-devel python-devel openssl-devel -``` - -Set up directory structure for build: -``` -mkdir -p ~/rpmbuild/ -cd ~/rpmbuild/ -mkdir -p BUILD RPMS SOURCES SPECS SRPMS -> SPECS/azure-cli.spec; vi SPECS/azure-cli.spec +sudo yum install -y gcc git rpm-build rpm-devel rpmlint make bash coreutils diffutils patch rpmdevtools python libffi-devel python-devel openssl-devel ``` -Set the CLI version and SHA256 for the archive: +Build example: +Note: use the full path to the repo path, not a relative path. ``` +git clone https://github.com/azure/azure-cli export CLI_VERSION=2.0.16 -export CLI_DOWNLOAD_SHA256=22c048d2911c13738c6b901a741ea655f277e0d9eb756c4fb9aee6bb6c2b0109 -``` - -RPM Build: -``` -rpmbuild -v -bb --clean SPECS/azure-cli.spec +export REPO_PATH=~/azure-cli +rpmbuild -v -bb --clean azure-cli/build_scripts/rpm/azure-cli.spec ``` Verification diff --git a/packaged_releases/rpm/azure-cli.spec b/build_scripts/rpm/azure-cli.spec similarity index 83% rename from packaged_releases/rpm/azure-cli.spec rename to build_scripts/rpm/azure-cli.spec index bc3882d16a4..c8a6315bfe7 100644 --- a/packaged_releases/rpm/azure-cli.spec +++ b/build_scripts/rpm/azure-cli.spec @@ -9,8 +9,7 @@ %define name azure-cli %define release 1%{?dist} %define version %{getenv:CLI_VERSION} -%define source_sha256 %{getenv:CLI_DOWNLOAD_SHA256} -%define source_url https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_%{version}.tar.gz +%define repo_path %{getenv:REPO_PATH} %define venv_url https://pypi.python.org/packages/source/v/virtualenv/virtualenv-15.0.0.tar.gz %define venv_sha256 70d63fb7e949d07aeb37f6ecc94e8b60671edb15b890aa86dba5dfaf2225dc19 %define cli_lib_dir %{_libdir}/az @@ -20,7 +19,6 @@ License: MIT Name: %{name} Version: %{version} Release: %{release} -Source0: %{source_url} Url: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli BuildArch: x86_64 Requires: python @@ -40,11 +38,6 @@ A great cloud needs great tools; we're excited to introduce Azure CLI 2.0, tmp_venv_archive=$(mktemp) tmp_source_archive=$(mktemp) -# Download, Extract Source -wget %{source_url} -qO $tmp_source_archive -echo "%{source_sha256} $tmp_source_archive" | sha256sum -c - -tar -xvzf $tmp_source_archive -C %{_builddir} - # Download, Extract Virtualenv wget %{venv_url} -qO $tmp_venv_archive echo "%{venv_sha256} $tmp_venv_archive" | sha256sum -c - @@ -55,13 +48,16 @@ tar -xvzf $tmp_venv_archive -C %{_builddir} python %{_builddir}/virtualenv-15.0.0/virtualenv.py --python python %{buildroot}%{cli_lib_dir} # Build the wheels from the source -source_dir=%{_builddir}/azure-cli_packaged_%{version} +source_dir=%{repo_path} dist_dir=$(mktemp -d) for d in $source_dir/src/azure-cli $source_dir/src/azure-cli-core $source_dir/src/azure-cli-nspkg $source_dir/src/azure-cli-command_modules-nspkg $source_dir/src/command_modules/azure-cli-*/; \ do cd $d; %{buildroot}%{cli_lib_dir}/bin/python setup.py bdist_wheel -d $dist_dir; cd -; done; +[ -d $source_dir/privates ] && cp $source_dir/privates/*.whl $dist_dir + # Install the CLI -%{buildroot}%{cli_lib_dir}/bin/pip install azure-cli --find-links $dist_dir +all_modules=`find $dist_dir -name "*.whl"` +%{buildroot}%{cli_lib_dir}/bin/pip install $all_modules %{buildroot}%{cli_lib_dir}/bin/pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg # Fix up %{buildroot} appearing in some files... diff --git a/build_scripts/wheels/README.md b/build_scripts/wheels/README.md new file mode 100644 index 00000000000..72ecc5390d0 --- /dev/null +++ b/build_scripts/wheels/README.md @@ -0,0 +1,9 @@ +Wheel Builds +============ + +For example: +``` +whl_dir=$(mktemp -d) +python build_scripts/wheels/build.py $whl_dir . +ls -la $whl_dir +``` diff --git a/packaged_releases/windows/scripts/build-packages.py b/build_scripts/wheels/build.py similarity index 100% rename from packaged_releases/windows/scripts/build-packages.py rename to build_scripts/wheels/build.py diff --git a/packaged_releases/windows/Product.wxs b/build_scripts/windows/Product.wxs similarity index 99% rename from packaged_releases/windows/Product.wxs rename to build_scripts/windows/Product.wxs index e4ec23b7be1..89732b81b48 100644 --- a/packaged_releases/windows/Product.wxs +++ b/build_scripts/windows/Product.wxs @@ -1,7 +1,7 @@ - + diff --git a/packaged_releases/windows/README.md b/build_scripts/windows/README.md similarity index 67% rename from packaged_releases/windows/README.md rename to build_scripts/windows/README.md index e14db5c0240..2826c557540 100644 --- a/packaged_releases/windows/README.md +++ b/build_scripts/windows/README.md @@ -18,6 +18,11 @@ Prerequisites 4. Install 'Microsoft Build Tools 2015'. https://www.microsoft.com/en-us/download/details.aspx?id=48159 +5. Install Python 3.6 + - https://www.python.org/ftp/python/3.6.1/python-3.6.1.exe + - python-installer.exe /quiet InstallAllUsers=0 TargetDir=%PYTHON_DIR% PrependPath=0 AssociateFiles=0 CompileAll=1 Shortcuts=0 Include_test=0 Include_doc=0 Include_dev=0 Include_launcher=0 Include_tcltk=0 Include_tools=0 + - %PYTHON_DIR% should be on PATH. + 5. Clone the repository. - git clone https://github.com/azure/azure-cli @@ -26,9 +31,8 @@ Note: The above can be done on a Windows 10 VM. Building -------- -1. Set the `CLIVERSION` environment variable. +1. Set the `CLI_VERSION` environment variable. -2. Run `build.cmd`. - - Can `cd packaged_releases\windows` first. +2. Run `build_scripts\windows\scripts\build.cmd`. 3. The unsigned MSI will be in the `.\out` folder. diff --git a/packaged_releases/windows/azure-cli.sln b/build_scripts/windows/azure-cli.sln similarity index 100% rename from packaged_releases/windows/azure-cli.sln rename to build_scripts/windows/azure-cli.sln diff --git a/packaged_releases/windows/azure-cli.wixproj b/build_scripts/windows/azure-cli.wixproj similarity index 100% rename from packaged_releases/windows/azure-cli.wixproj rename to build_scripts/windows/azure-cli.wixproj diff --git a/packaged_releases/windows/propagate_env_change/main.cpp b/build_scripts/windows/propagate_env_change/main.cpp similarity index 100% rename from packaged_releases/windows/propagate_env_change/main.cpp rename to build_scripts/windows/propagate_env_change/main.cpp diff --git a/packaged_releases/windows/propagate_env_change/propagate_env_change.exe b/build_scripts/windows/propagate_env_change/propagate_env_change.exe similarity index 100% rename from packaged_releases/windows/propagate_env_change/propagate_env_change.exe rename to build_scripts/windows/propagate_env_change/propagate_env_change.exe diff --git a/packaged_releases/windows/propagate_env_change/propagate_env_change.sln b/build_scripts/windows/propagate_env_change/propagate_env_change.sln similarity index 100% rename from packaged_releases/windows/propagate_env_change/propagate_env_change.sln rename to build_scripts/windows/propagate_env_change/propagate_env_change.sln diff --git a/packaged_releases/windows/propagate_env_change/propagate_env_change.vcxproj b/build_scripts/windows/propagate_env_change/propagate_env_change.vcxproj similarity index 100% rename from packaged_releases/windows/propagate_env_change/propagate_env_change.vcxproj rename to build_scripts/windows/propagate_env_change/propagate_env_change.vcxproj diff --git a/packaged_releases/windows/propagate_env_change/propagate_env_change.vcxproj.filters b/build_scripts/windows/propagate_env_change/propagate_env_change.vcxproj.filters similarity index 100% rename from packaged_releases/windows/propagate_env_change/propagate_env_change.vcxproj.filters rename to build_scripts/windows/propagate_env_change/propagate_env_change.vcxproj.filters diff --git a/packaged_releases/windows/resources/CLI_LICENSE.rtf b/build_scripts/windows/resources/CLI_LICENSE.rtf similarity index 100% rename from packaged_releases/windows/resources/CLI_LICENSE.rtf rename to build_scripts/windows/resources/CLI_LICENSE.rtf diff --git a/packaged_releases/windows/resources/ThirdPartyNotices.txt b/build_scripts/windows/resources/ThirdPartyNotices.txt similarity index 100% rename from packaged_releases/windows/resources/ThirdPartyNotices.txt rename to build_scripts/windows/resources/ThirdPartyNotices.txt diff --git a/packaged_releases/windows/resources/banner.bmp b/build_scripts/windows/resources/banner.bmp similarity index 100% rename from packaged_releases/windows/resources/banner.bmp rename to build_scripts/windows/resources/banner.bmp diff --git a/packaged_releases/windows/resources/dialog.bmp b/build_scripts/windows/resources/dialog.bmp similarity index 100% rename from packaged_releases/windows/resources/dialog.bmp rename to build_scripts/windows/resources/dialog.bmp diff --git a/packaged_releases/windows/resources/windowsazure.ico b/build_scripts/windows/resources/windowsazure.ico similarity index 100% rename from packaged_releases/windows/resources/windowsazure.ico rename to build_scripts/windows/resources/windowsazure.ico diff --git a/packaged_releases/windows/scripts/az.cmd b/build_scripts/windows/scripts/az.cmd similarity index 100% rename from packaged_releases/windows/scripts/az.cmd rename to build_scripts/windows/scripts/az.cmd diff --git a/packaged_releases/windows/scripts/build_local.cmd b/build_scripts/windows/scripts/build.cmd similarity index 69% rename from packaged_releases/windows/scripts/build_local.cmd rename to build_scripts/windows/scripts/build.cmd index b60f31cfdbf..9c564445d5c 100644 --- a/packaged_releases/windows/scripts/build_local.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -3,10 +3,10 @@ SetLocal EnableDelayedExpansion echo build a msi installer using local cli sources and python executables. You need to have curl.exe, unzip.exe and msbuild.exe available under PATH echo. -set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin" +set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;" -if "%CLIVERSION%"=="" ( - echo Please set the CLIVERSION environment variable, e.g. 2.0.13 +if "%CLI_VERSION%"=="" ( + echo Please set the CLI_VERSION environment variable, e.g. 2.0.13 goto ERROR ) set PYTHON_VERSION=3.6.1 @@ -66,7 +66,9 @@ if exist %TEMP_SCRATCH_FOLDER% ( ) mkdir %TEMP_SCRATCH_FOLDER% -copy %REPO_ROOT%\privates\*.whl %TEMP_SCRATCH_FOLDER% +if exist %REPO_ROOT%\privates ( + copy %REPO_ROOT%\privates\*.whl %TEMP_SCRATCH_FOLDER% +) ::ensure wix is available if exist %WIX_DIR% ( @@ -88,25 +90,47 @@ if not exist %WIX_DIR% ( robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL :: Build & install all the packages with bdist_wheel -%BUILDING_DIR%\python %~dp0build-packages.py %TEMP_SCRATCH_FOLDER% %REPO_ROOT% +%BUILDING_DIR%\python.exe -m pip install wheel +echo Building CLI packages... +set CLI_SRC=%REPO_ROOT%\src +for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg) do ( + pushd %%a + %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %TEMP_SCRATCH_FOLDER% + popd +) +pushd %CLI_SRC%\command_modules +for /D %%a in (*) do ( + pushd %CLI_SRC%\command_modules\%%a + %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %TEMP_SCRATCH_FOLDER% + popd +) +popd +echo Built CLI packages successfully. + if %errorlevel% neq 0 goto ERROR -:: Install them to the temp folder so to be packaged -%BUILDING_DIR%\python.exe -m pip install -f %TEMP_SCRATCH_FOLDER% --no-cache-dir azure-cli + + +set ALL_MODULES= +for %%i in (%TEMP_SCRATCH_FOLDER%\*.whl) do ( + set ALL_MODULES=!ALL_MODULES! %%i +) +echo All modules: %ALL_MODULES% +%BUILDING_DIR%\python.exe -m pip install --no-cache-dir %ALL_MODULES% %BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg echo Creating the wbin (Windows binaries) folder that will be added to the path... mkdir %BUILDING_DIR%\wbin -copy %REPO_ROOT%\packaged_releases\windows\scripts\az.cmd %BUILDING_DIR%\wbin\ +copy %REPO_ROOT%\build_scripts\windows\scripts\az.cmd %BUILDING_DIR%\wbin\ if %errorlevel% neq 0 goto ERROR -copy %REPO_ROOT%\packaged_releases\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% -copy %REPO_ROOT%\packaged_releases\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% +copy %REPO_ROOT%\build_scripts\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR% +copy %REPO_ROOT%\build_scripts\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR% del %BUILDING_DIR%\Scripts\pip.exe del %BUILDING_DIR%\Scripts\pip3.exe del %BUILDING_DIR%\Scripts\pip3.6.exe if %errorlevel% neq 0 goto ERROR echo Building MSI... -msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\packaged_releases\windows\azure-cli.wixproj +msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\build_scripts\windows\azure-cli.wixproj start %OUTPUT_DIR% diff --git a/packaged_releases/README.md b/packaged_releases/README.md deleted file mode 100644 index 148f0189be7..00000000000 --- a/packaged_releases/README.md +++ /dev/null @@ -1,76 +0,0 @@ -Creating a Packaged Release -=========================== - -This document provides instructions on creating a packaged release. - -Throughout this document, `{VERSION}` refers to a semantic version for this packaged release (e.g. `0.2.3`). - - -1 - Create release archive --------------------------- - -We create a `.tar.gz` containing the source code for the packaged release. -This archive will be used as the basis for the Docker, Debian and Homebrew builds as they build from this source. - -Clone the repo afresh: -``` -$ git clone https://github.com/azure/azure-cli -``` - -Run the script to create the release archive from the 'scripts' folder at the repo root: -``` -cd scripts -python -m automation.release.packaged --version {VERSION} --components azure-cli=VERSION ... -``` - -A full example: -``` -cat ~/cli-components.json -{ - "azure-cli": "2.0.1", - "azure-cli-core": "2.0.1", - "azure-cli-component": "2.0.0", - "azure-cli-acs": "2.0.0" -} -python -m automation.release.packaged --version 0.2.3 -f ~/cli-components.json -``` - -OR - -``` -python -m automation.release.packaged --version 2.0.9 --components azure-cli=2.0.9 azure-cli-acr=2.0.7 azure-cli-acs=2.0.9 azure-cli-appservice=0.1.9 azure-cli-batch=3.0.2 azure-cli-billing=0.1.2 azure-cli-cdn=0.0.5 azure-cli-cloud=2.0.5 azure-cli-cognitiveservices=0.1.5 azure-cli-command_modules-nspkg=2.0.0 azure-cli-component=2.0.6 azure-cli-configure=2.0.9 azure-cli-consumption=0.1.2 azure-cli-core=2.0.10 azure-cli-cosmosdb=0.1.9 azure-cli-dla=0.0.9 azure-cli-dls=0.0.9 azure-cli-feedback=2.0.5 azure-cli-find=0.2.5 azure-cli-interactive=0.3.5 azure-cli-iot=0.1.8 azure-cli-keyvault=2.0.7 azure-cli-lab=0.0.7 azure-cli-monitor=0.0.7 azure-cli-network=2.0.9 azure-cli-nspkg=3.0.0 azure-cli-profile=2.0.7 azure-cli-rdbms=0.0.4 azure-cli-redis=0.2.6 azure-cli-resource=2.0.9 azure-cli-role=2.0.7 azure-cli-sql=2.0.6 azure-cli-storage=2.0.9 azure-cli-vm=2.0.9 -``` - -2 - Upload release archive --------------------------- - -The release archive should be uploaded to a storage account. - -Upload the archive: -``` -export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -g azure-cli-prod -n azurecliprod -otsv) -az storage blob upload -f azure-cli_packaged_{VERSION}.tar.gz -n azure-cli_packaged_{VERSION}.tar.gz -c releases -``` - -Get the URL for the uploaded archive: -``` -az storage blob url -c releases -n azure-cli_packaged_{VERSION}.tar.gz -``` - -An example URL is `https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_{VERSION}.tar.gz`. - -Get the SHA256 checksum: -``` -shasum -a 256 azure-cli_packaged_{VERSION}.tar.gz -``` - -3 - Build/Release for Debian, Docker, Windows, RPM, Homebrew ------------------------------------------------------------- - -Follow the instructions in the `debian`, `docker`, `windows`, `rpm` and `homebrew` subdirectories to create these releases. - - -4 - Modify HISTORY.md ---------------------- - -Modify the packaged release history with release notes on this release and create a PR for this change. diff --git a/packaged_releases/docker/Dockerfile b/packaged_releases/docker/Dockerfile deleted file mode 100644 index 9a2f651e97e..00000000000 --- a/packaged_releases/docker/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -#--------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -#--------------------------------------------------------------------------------------------- - -FROM python:3.5.2-alpine - -ARG CLI_VERSION -ARG CLI_DOWNLOAD_SHA256 - -# Metadata as defined at http://label-schema.org -ARG BUILD_DATE -LABEL org.label-schema.schema-version="1.0" \ - org.label-schema.vendor="Microsoft" \ - org.label-schema.name="Azure CLI 2.0" \ - org.label-schema.version=$CLI_VERSION \ - org.label-schema.license="MIT" \ - org.label-schema.description="The Azure CLI 2.0 is the new Azure CLI and is applicable when you use the Resource Manager deployment model." \ - org.label-schema.url="https://docs.microsoft.com/en-us/cli/azure/overview" \ - org.label-schema.usage="https://docs.microsoft.com/en-us/cli/azure/install-az-cli2#docker" \ - org.label-schema.build-date=$BUILD_DATE \ - org.label-schema.vcs-url="https://github.com/Azure/azure-cli.git" \ - org.label-schema.docker.cmd="docker run -v ${HOME}:/root -it azuresdk/azure-cli-python:" - -# INSTALL DEPENDENCIES -# pip wheel - required for CLI packaging -# jmespath-terminal - we include jpterm as a useful tool -RUN pip install --no-cache-dir --upgrade pip wheel jmespath-terminal -# bash gcc openssl-dev libffi-dev musl-dev - dependencies required for CLI -# jq - we include jq as a useful tool -# openssh - included for ssh-keygen -# ca-certificates -# wget - required for installing jp -RUN apk add --no-cache bash gcc make openssl-dev libffi-dev musl-dev jq openssh ca-certificates wget openssl git \ - && update-ca-certificates -# We also, install jp -RUN wget https://github.com/jmespath/jp/releases/download/0.1.2/jp-linux-amd64 -qO /usr/local/bin/jp \ - && chmod +x /usr/local/bin/jp - -# DOWNLOAD, EXTRACT, PATCH, BUILD, INSTALL CLI -# Download and extract -RUN mkdir /cli-src -RUN src_zip=$(mktemp) \ - && wget https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_${CLI_VERSION}.tar.gz -qO $src_zip \ - && echo "$CLI_DOWNLOAD_SHA256 $src_zip" | sha256sum -c - \ - && tar -xvzf $src_zip -C /cli-src \ - && rm $src_zip -# Build and install -RUN /bin/bash -c 'cd /cli-src/*; tmp_pkg_dir=$(mktemp -d); \ - for d in src/azure-cli src/azure-cli-core src/azure-cli-nspkg src/azure-cli-command_modules-nspkg src/command_modules/azure-cli-*/; \ - do cd $d; python setup.py bdist_wheel -d $tmp_pkg_dir; cd -; \ - done; \ - pip install azure-cli -f $tmp_pkg_dir; \ - pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg;' -# Tab completion -RUN cat /cli-src/*/az.completion >> ~/.bashrc - -RUN rm -rf /cli-src - -WORKDIR / - -CMD bash diff --git a/packaged_releases/windows/build.cmd b/packaged_releases/windows/build.cmd deleted file mode 100644 index 1af1d7b8e71..00000000000 --- a/packaged_releases/windows/build.cmd +++ /dev/null @@ -1,22 +0,0 @@ -@echo off -echo Building the Windows Installer... -echo. -:: Add Git to the path as this should be run through a .NET command prompt -:: and not a Git bash shell... We also need the gnu toolchain (for curl & unzip) -set "PATH=%PATH%;C:\Program Files (x86)\Git\bin;C:\Program Files (x86)\MSBuild\14.0\Bin;" - -pushd %~dp0 - -CALL scripts\prepareBuild.cmd -if %errorlevel% neq 0 exit /b %errorlevel% - -echo Building MSI... -where msbuild -msbuild /t:rebuild /p:Configuration=Release - -echo. - -if not "%1"=="-noprompt" ( - start .\out\ -) -popd diff --git a/packaged_releases/windows/scripts/prepareBuild.cmd b/packaged_releases/windows/scripts/prepareBuild.cmd deleted file mode 100644 index 5b729b7fed3..00000000000 --- a/packaged_releases/windows/scripts/prepareBuild.cmd +++ /dev/null @@ -1,161 +0,0 @@ -@echo off -:: Microsoft Azure CLI - Windows Installer - Author file components script -:: Copyright (C) Microsoft Corporation. All Rights Reserved. -:: -:: This re-builds partial WiX files for use in cloning the repo after install. -:: heat.exe from the WiX toolset is used for this. -:: -if "%CLIVERSION%"=="" ( - echo "Set the CLIVERSION environment variable." - goto ERROR -) - -::when change to a later version, please update ones in build_local.cmd -set PYTHON_VERSION=3.6.1 - -pushd %~dp0..\ - -set CLI_ARCHIVE_DOWNLOAD_URL=https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_%CLIVERSION%.tar.gz - -:: Download URL for Wix 10 from https://wix.codeplex.com/downloads/get/1587180 -:: Direct download URL http://download-codeplex.sec.s-msft.com/Download/Release?ProjectName=wix&DownloadId=1587180&FileTime=131118854877130000&Build=21050 -:: We use a mirror of Wix storage on Azure blob storage as the above link can be slow... -set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" - -set PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python/%PYTHON_VERSION%/python-%PYTHON_VERSION%.exe - -echo Downloading Python from %PYTHON_DOWNLOAD_URL% - -:: Set up the output directory and temp. directories -echo Cleaning previous build artifacts... -set OUTPUT_DIR=.\out -if exist %OUTPUT_DIR% rmdir /s /q %OUTPUT_DIR% -mkdir %OUTPUT_DIR% - -set TEMP_CLI_FOLDER=zcli -set TEMP_SCRATCH_FOLDER=zcli_scratch -set TEMP_WIX_FOLDER=zwix -set TEMP_PYTHON_FOLDER=zPython -set BUILDING_DIR=%HOMEDRIVE%%HOMEPATH%\%TEMP_CLI_FOLDER% -set SCRATCH_DIR=%HOMEDRIVE%%HOMEPATH%\%TEMP_SCRATCH_FOLDER% -set WIX_DIR=%HOMEDRIVE%%HOMEPATH%\%TEMP_WIX_FOLDER% -set PYTHON_DIR=%HOMEDRIVE%%HOMEPATH%\%TEMP_PYTHON_FOLDER% - -pushd %HOMEDRIVE%%HOMEPATH% -if exist %TEMP_CLI_FOLDER% rmdir /s /q %TEMP_CLI_FOLDER% -if exist %TEMP_WIX_FOLDER% rmdir /s /q %TEMP_WIX_FOLDER% -::rmdir always returns 0, so check folder's existence -if exist %TEMP_CLI_FOLDER% ( - echo Failed to delete %TEMP_CLI_FOLDER%. - goto ERROR -) -mkdir %TEMP_CLI_FOLDER% - -if exist %TEMP_SCRATCH_FOLDER% rmdir /s /q %TEMP_SCRATCH_FOLDER% -if exist %TEMP_SCRATCH_FOLDER% ( - echo Failed to delete %TEMP_SCRATCH_FOLDER%. - goto ERROR -) -mkdir %TEMP_SCRATCH_FOLDER% - -if exist %TEMP_PYTHON_FOLDER% ( - echo Using existing Python at %PYTHON_DIR% -) -if not exist %TEMP_PYTHON_FOLDER% ( - mkdir %TEMP_PYTHON_FOLDER% - pushd %PYTHON_DIR% - echo Downloading Python %PYTHON_VERSION% - curl -o python-installer.exe %PYTHON_DOWNLOAD_URL% -k - python-installer.exe /quiet InstallAllUsers=0 TargetDir=%PYTHON_DIR% PrependPath=0 AssociateFiles=0 CompileAll=1 Shortcuts=0 Include_test=0 Include_doc=0 Include_dev=0 Include_launcher=0 Include_tcltk=0 Include_tools=0 - if %errorlevel% neq 0 goto ERROR - del python-installer.exe - echo Downloaded Python %PYTHON_VERSION% to %PYTHON_DIR% successfully. - popd -) - -if exist %TEMP_WIX_FOLDER% ( - echo Using existing Wix at %WIX_DIR% -) -if not exist %TEMP_WIX_FOLDER% ( - mkdir %TEMP_WIX_FOLDER% - pushd %WIX_DIR% - echo Downloading Wix. - curl -o wix-archive.zip %WIX_DOWNLOAD_URL% -k - unzip -q wix-archive.zip - del wix-archive.zip - if %errorlevel% neq 0 goto ERROR - echo Wix downloaded and extracted successfully. - popd -) - -popd - -:: Download & unzip CLI archive -pushd %BUILDING_DIR% -echo Downloading CLI archive version %CLIVERSION% -curl -o cli-archive.tar.gz %CLI_ARCHIVE_DOWNLOAD_URL% -k -curl -o 7zipsetup.exe http://www.7-zip.org/a/7z1604-x64.exe -k -7zipsetup.exe /S -"C:\Program Files\7-Zip\7z.exe" x -r cli-archive.tar.gz -"C:\Program Files\7-Zip\7z.exe" x -r azure-cli_packaged_%CLIVERSION%.tar -echo Extracted. -del 7zipsetup.exe -del cli-archive.tar.gz -del azure-cli_packaged_%CLIVERSION%.tar -if %errorlevel% neq 0 goto ERROR -echo Downloaded and extracted CLI archive successfully. -popd - -echo Python directory info... -for %%i in (%PYTHON_DIR%) do echo %%i - -:: Use the Python version on the machine that creates the MSI -robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL - -:: Build & install all the packages with bdist_wheel -%BUILDING_DIR%\python.exe -m pip install wheel -echo Building CLI packages... -set CLI_SRC=%BUILDING_DIR%\azure-cli_packaged_%CLIVERSION%\src -for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nspkg) do ( - pushd %%a - %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %SCRATCH_DIR% - popd -) -pushd %CLI_SRC%\command_modules -for /D %%a in (*) do ( - pushd %CLI_SRC%\command_modules\%%a - %BUILDING_DIR%\python.exe setup.py bdist_wheel -d %SCRATCH_DIR% - popd -) -echo Built CLI packages successfully. -popd -%BUILDING_DIR%\python.exe -m pip install azure-cli -f %SCRATCH_DIR% -%BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg - -rmdir /s /q %BUILDING_DIR%\azure-cli_packaged_%CLIVERSION% - -echo Creating the wbin (Windows binaries) folder that will be added to the path... -mkdir %BUILDING_DIR%\wbin -copy .\scripts\az.cmd %BUILDING_DIR%\wbin\ -if %errorlevel% neq 0 goto ERROR -copy .\resources\CLI_LICENSE.rtf %BUILDING_DIR% -copy .\resources\ThirdPartyNotices.txt %BUILDING_DIR% -del %BUILDING_DIR%\Scripts\pip.exe -del %BUILDING_DIR%\Scripts\pip3.exe -del %BUILDING_DIR%\Scripts\pip3.6.exe -if %errorlevel% neq 0 goto ERROR - -echo. - -:SUCCESS -echo Looks good. - -goto END - -:ERROR -echo Error occurred, please check the output for details. -exit /b 1 - -:END -exit /b 0 -popd diff --git a/scripts/ci/test_automation.sh b/scripts/ci/test_automation.sh index 0aad310a079..d5e50209b78 100755 --- a/scripts/ci/test_automation.sh +++ b/scripts/ci/test_automation.sh @@ -10,6 +10,7 @@ ALL_MODULES=`find $share_folder/build/ -name "*.whl"` pip install -qqq -e ./tools pip install -qqq coverage codecov +[ -d privates ] && pip install -qqq privates/*.whl pip install -qqq $ALL_MODULES echo '=== List installed packages' diff --git a/scripts/ci/test_static.sh b/scripts/ci/test_static.sh index 202168d12bb..6b21ef1f66b 100755 --- a/scripts/ci/test_static.sh +++ b/scripts/ci/test_static.sh @@ -8,6 +8,7 @@ ls -la $share_folder/build ALL_MODULES=`find $share_folder/build/ -name "*.whl"` +[ -d privates ] && pip install privates/*.whl pip install pylint flake8 pip install $ALL_MODULES diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py index ffea4926ba1..3f0e7b249ad 100755 --- a/scripts/dev_setup.py +++ b/scripts/dev_setup.py @@ -26,6 +26,12 @@ def exec_command(command): print('Running dev setup...') print('Root directory \'{}\'\n'.format(root_dir)) +# install private whls if there are any +privates_dir = os.path.join(root_dir, 'privates') +if os.path.isdir(privates_dir) and os.listdir(privates_dir): + whl_list = ' '.join([os.path.join(privates_dir, f) for f in os.listdir(privates_dir)]) + exec_command('pip install {}'.format(whl_list)) + # install general requirements exec_command('pip install -r requirements.txt') diff --git a/scripts/releaser/Dockerfile b/scripts/releaser/Dockerfile deleted file mode 100644 index e2886d137b7..00000000000 --- a/scripts/releaser/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -FROM python:3.5.2-alpine - -RUN apk upgrade --no-cache && \ - apk add --no-cache bash git openssh gcc make nano vim \ - openssl-dev libffi-dev musl-dev ca-certificates openssl && update-ca-certificates - -RUN pip install --no-cache-dir --upgrade pip wheel twine requests virtualenv uritemplate.py azure-cli sh - -ADD . / - -CMD cat README.md; printf "\n\n** starting bash...\n\n"; bash diff --git a/scripts/releaser/HISTORY.rst b/scripts/releaser/HISTORY.rst deleted file mode 100644 index 897878d18d4..00000000000 --- a/scripts/releaser/HISTORY.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. :changelog: - -Release History -=============== - - -0.1.6 (2017-09-25) -++++++++++++++++++ - -* Script to create new Homebrew formula for Azure CLI that PRs can be created from. -* Fix release.py to use 'tools' instead of 'scripts' as the internal package was moved. - -0.1.5 (2017-09-11) -++++++++++++++++++ - -* Fix RPM script to print the Python upload script instead of automatically uploading as we need to sign first. - -0.1.4 (2017-09-07) -++++++++++++++++++ - -* Add release script for RPM releases. - -0.1.3 (2017-08-18) -++++++++++++++++++ - -* Fix 'packaged release archive' creation step. We now clone the repo again after pushing tags so we can use the new tags. - -0.1.2 (2017-08-15) -++++++++++++++++++ - -* Fix Debian release script failing due to double quotes in printing the status url after the build. - -0.1.1 (2017-07-31) -++++++++++++++++++ - -* Support releasing a module that hasn't been released before. -* Support Docker password with special characters. - -0.1.0 (2017-07-05) -++++++++++++++++++ - -* First release. -* Push modules to Git. -* Publish all modules to PyPI. -* Create GitHub releases. -* Create and Publish packaged release archive. -* Create and Publish Docker image. -* Create and Publish Debian package. diff --git a/scripts/releaser/README.md b/scripts/releaser/README.md deleted file mode 100644 index 2a7c6979a9f..00000000000 --- a/scripts/releaser/README.md +++ /dev/null @@ -1,61 +0,0 @@ -Automated PyPI and GitHub releases of all CLI modules -===================================================== - -Description ------------ -This is a Docker image that automates releases of all CLI modules to PyPI and then creates GitHub releases for each. -The scripts have been tested on Python 3 so it's recommended to run the Docker image. - -How to Build ------------- -``` -sudo docker build --no-cache -t azuresdk/azure-cli-release-automation: . -``` - -How to Run ----------- -``` -sudo docker run -it -e "REPO_NAME=azure/azure-cli" -e "GITHUB_USER=user1" -e "GITHUB_USER_TOKEN=" \ --e "PYPI_REPO=https://test.pypi.org/legacy/" -e "TWINE_USERNAME=" -e "TWINE_PASSWORD=" \ --e "CLI_VERSION=0.0.0a1" -e "AZURE_STORAGE_CONNECTION_STRING=" \ - azuresdk/azure-cli-release-automation: -``` - -Once the container has started, there are several scripts available. -They each require they're own set of environment variables. -These can be set in the initial `docker run` command above or by using `export ENV=VALUE` directly in the running container. - -``` -python release.py -python release-docker.py -python release-debian.py -python release-rpm.py -python release-homebrew.py -``` - -Environment Variables ---------------------- -`REPO_NAME` - The name of the GitHub repo (e.g. azure/azure-cli) -`GITHUB_USER` - User id of the bot that will post comments and create releases. -`GITHUB_USER_TOKEN` - Access token for this user. -`PYPI_REPO` - URL to PyPI (e.g. https://test.pypi.org/legacy/ or https://upload.pypi.org/legacy/). -`TWINE_USERNAME` - Username to authenticate with PyPI. -`TWINE_PASSWORD` - Password to authenticate with PyPI. -`CLI_VERSION` - The new version of the CLI (used for packaged releases) -`AZURE_STORAGE_CONNECTION_STRING` - The Azure storage connection string to upload release assets - -The `GITHUB_USER` should have the following GitHub OAuth scopes: -- repo_deployment (to create GitHub releases and commit to master) - -`CLI_DOWNLOAD_SHA256` - The SHA256 sum of the packaged release (produced by `release.py`). - -`DOCKER_REPO` - The Docker repo to push the image to (e.g. azuresdk/azure-cli-python). -`DOCKER_USERNAME` - The Docker username that has push permissions to the above Docker repo. -`DOCKER_PASSWORD` - The Docker password for the user. - -`MS_REPO_URL` - The repository URL to publish .deb/.rpm packages. -`MS_REPO_USERNAME` - The repository username to publish .deb/.rpm packages. -`MS_REPO_PASSWORD` - The user password to publish the .deb/.rpm package. - -`DEBIAN_REPO_ID` - The repository ID to publish the .deb package. -`YUM_REPO_ID` - The repository ID to publish the .rpm package. diff --git a/scripts/releaser/release-debian.py b/scripts/releaser/release-debian.py deleted file mode 100644 index b81b0532bcb..00000000000 --- a/scripts/releaser/release-debian.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# This script is interactive as you need to log in to 'az'. - -from __future__ import print_function - -import os -import sys -import time -from datetime import datetime -from six import StringIO -from sh import az, ssh - - -script_env = {} - -def add_script_env(name): - script_env[name] = os.environ.get(name) - -add_script_env('REPO_NAME') -add_script_env('CLI_VERSION') -add_script_env('CLI_DOWNLOAD_SHA256') -add_script_env('AZURE_STORAGE_CONNECTION_STRING') -add_script_env('DEBIAN_REPO_ID') -add_script_env('MS_REPO_URL') -add_script_env('MS_REPO_USERNAME') -add_script_env('MS_REPO_PASSWORD') - -assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set. {}".format(script_env) - -REPO_UPLOAD_SCRIPT_TMPL = """ -import os, requests -payload = {{'name': 'azure-cli', 'version': '{cli_version}-1', 'repositoryId': '{repo_id}', 'sourceUrl': '{source_url}'}} -r = requests.post('{repo_package_url}', verify=False, auth=('{repo_user}', '{repo_pass}'), json=payload) -print('Status Code') -print(r.status_code) -print('Query with a GET to the following:') -print(r.headers['Location']) -""" - -def print_env_vars(): - for n in script_env: - print('{} = {}'.format(n, script_env[n])) - - -def print_status(msg=''): - print('-- '+msg) - -def print_heading(heading): - print('{0}\n{1}\n{0}'.format('=' * len(heading), heading)) - -def give_chance_to_cancel(msg_prefix=''): - cancel_time_secs = 10 - msg_tmpl = '{}: Starting in {} seconds.' - for i in range(cancel_time_secs, 0, -1): - print_status(msg_tmpl.format(msg_prefix, i)) - time.sleep(1) - -def main(): - print_env_vars() - time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S') - az(["login"], _out=sys.stdout, _err=sys.stdout) - resource_group = 'azurecli-release-debian-' + time_str - vm_name = 'vm-debian-' + time_str - print_status('Creating resource group.') - az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout) - print_status('Creating VM.') - az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh', - '--image', 'Canonical:UbuntuServer:14.04.4-LTS:latest', '--admin-username', 'ubuntu'], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - print_status('Getting VM IP address.') - az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name, - '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io) - ip_address = io.getvalue().strip().replace('"', '') - print_status('VM IP address is {}'.format(ip_address)) - io.close() - vm_connect_str = "ubuntu@{}".format(ip_address) - my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str]) - print_status('Installing git.') - my_vm(['sudo', 'apt-get', 'update', '&&', 'sudo', 'apt-get', 'install', '-y', 'git'], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - my_vm(['mktemp', '-d'], _out=io) - repo_dir = io.getvalue().strip() - io.close() - print_status('Cloning repo.') - my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout) - path_to_debian_build_script = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_build.sh') - path_to_dir_creator = os.path.join(repo_dir, 'packaged_releases', 'debian', 'debian_dir_creator.sh') - io = StringIO() - my_vm(['mktemp', '-d'], _out=io) - build_artifact_dir = io.getvalue().strip() - io.close() - print_status('Running debian build scripts.') - my_vm(['chmod', '+x', path_to_debian_build_script, path_to_dir_creator], _out=sys.stdout, _err=sys.stdout) - my_vm(['export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&', - 'export', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), '&&', - 'export', 'BUILD_ARTIFACT_DIR={}'.format(build_artifact_dir), '&&', - path_to_debian_build_script, path_to_dir_creator], - _out=sys.stdout, _err=sys.stdout) - print_status('Debian build complete.') - io = StringIO() - my_vm(['ls', build_artifact_dir], _out=io) - artifact_name = io.getvalue().strip() - io.close() - deb_file_path = os.path.join(build_artifact_dir, artifact_name) - print_status('Installing the .deb on the build machine') - my_vm(['sudo', 'dpkg', '-i', deb_file_path], _out=sys.stdout, _err=sys.stdout) - # Upload to Azure Storage - print_status('Uploading .deb to Azure storage.') - my_vm(['az', 'storage', 'container', 'create', '--name', 'repos', '--public-access', 'blob', - '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], - _out=sys.stdout, _err=sys.stdout) - my_vm(['az', 'storage', 'blob', 'upload', '-f', deb_file_path, - '-n', artifact_name, '-c', 'repos', '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - my_vm(['az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'repos', '--output', 'tsv', - '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], _out=io) - deb_url = io.getvalue().strip() - io.close() - print_status('Debian file uploaded to the following URL.') - print_status(deb_url) - # Publish to apt service - my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout) - my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout) - my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout) - upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'), - repo_id=script_env.get('DEBIAN_REPO_ID'), - source_url=deb_url, - repo_package_url=script_env.get('MS_REPO_URL'), - repo_user=script_env.get('MS_REPO_USERNAME'), - repo_pass=script_env.get('MS_REPO_PASSWORD')) - my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) - my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) - print_status('Done. :)') - give_chance_to_cancel('Delete resource group (in background)') - az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout) - print_status('Finished. :)') - -if __name__ == '__main__': - main() diff --git a/scripts/releaser/release-docker.py b/scripts/releaser/release-docker.py deleted file mode 100644 index 245ea501837..00000000000 --- a/scripts/releaser/release-docker.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# This script is interactive as you need to log in to 'az'. - -from __future__ import print_function - -import os -import sys -import time -from datetime import datetime -from six import StringIO -from sh import az, ssh - -script_env = {} - -def add_script_env(name): - script_env[name] = os.environ.get(name) - -add_script_env('REPO_NAME') -add_script_env('CLI_VERSION') -add_script_env('CLI_DOWNLOAD_SHA256') -add_script_env('DOCKER_REPO') -add_script_env('DOCKER_USERNAME') -add_script_env('DOCKER_PASSWORD') - -assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set. {}".format(script_env) - -def print_env_vars(): - for n in script_env: - print('{} = {}'.format(n, script_env[n])) - -def print_status(msg=''): - print('-- '+msg) - -def print_heading(heading): - print('{0}\n{1}\n{0}'.format('=' * len(heading), heading)) - -def give_chance_to_cancel(msg_prefix=''): - cancel_time_secs = 10 - msg_tmpl = '{}: Starting in {} seconds.' - for i in range(cancel_time_secs, 0, -1): - print_status(msg_tmpl.format(msg_prefix, i)) - time.sleep(1) - -def main(): - print_env_vars() - time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S') - az(["login"], _out=sys.stdout, _err=sys.stdout) - resource_group = 'azurecli-release-docker-' + time_str - vm_name = 'vm-docker-' + time_str - print_status('Creating resource group.') - az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout) - print_status('Creating VM.') - az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh', - '--image', 'Canonical:UbuntuServer:16.04-LTS:latest', '--admin-username', 'ubuntu'], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - print_status('Getting VM IP address.') - az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name, - '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io) - ip_address = io.getvalue().strip().replace('"', '') - print_status('VM IP address is {}'.format(ip_address)) - io.close() - vm_connect_str = "ubuntu@{}".format(ip_address) - my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str]) - print_status('Installing Docker.') - my_vm(['curl', '-sSL', 'https://get.docker.com/', '-o', 'docker_install_script.sh'], - _out=sys.stdout, _err=sys.stdout) - my_vm(['sh', 'docker_install_script.sh'], _out=sys.stdout, _err=sys.stdout) - print_status('Docker installed.') - io = StringIO() - my_vm(['mktemp', '-d'], _out=io) - repo_dir = io.getvalue().strip() - io.close() - print_status('Cloning repo.') - my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout) - image_tag = '{}:{}'.format(script_env.get('DOCKER_REPO'), script_env.get('CLI_VERSION')) - path_to_dockerfile = os.path.join(repo_dir, 'packaged_releases', 'docker', 'Dockerfile') - path_to_docker_context = os.path.join(repo_dir, 'packaged_releases', 'docker') - print_status('Running Docker build.') - my_vm(['sudo', 'docker', 'build', '--no-cache', - '--build-arg', 'BUILD_DATE="`date -u +"%Y-%m-%dT%H:%M:%SZ"`"', - '--build-arg', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), - '--build-arg', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), - '-f', path_to_dockerfile, - '-t', image_tag, - path_to_docker_context], _out=sys.stdout, _err=sys.stdout) - print_status('Docker build complete.') - print_status('Running Docker log in.') - my_vm(['sudo', 'docker', 'login', '--username', script_env.get('DOCKER_USERNAME'), '--password', '"{}"'.format(script_env.get('DOCKER_PASSWORD'))], - _out=sys.stdout, _err=sys.stdout) - print_status('Running Docker push.') - my_vm(['sudo', 'docker', 'push', image_tag], _out=sys.stdout, _err=sys.stdout) - print_status('Image pushed to Docker Hub.') - print_status('Done. :)') - give_chance_to_cancel('Delete resource group (in background)') - az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout) - print_status('Finished. :)') - -if __name__ == '__main__': - main() diff --git a/scripts/releaser/release-homebrew.py b/scripts/releaser/release-homebrew.py deleted file mode 100644 index 1501b698804..00000000000 --- a/scripts/releaser/release-homebrew.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -import os -import sys -import tempfile -from collections import OrderedDict -from sh import pip, git - -HOMEBREW_REPO_URL = 'https://github.com/Homebrew/homebrew-core' -AZURE_CLI_FORMULA_PATH = os.path.join('Formula', 'azure-cli.rb') - -UPSTREAM_URL_TMPL = 'https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_{cli_version}.tar.gz' - -script_env = {} - -def add_script_env(name): - script_env[name] = os.environ.get(name) - -add_script_env('CLI_VERSION') -add_script_env('CLI_DOWNLOAD_SHA256') - -assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set. {}".format(script_env) - -def print_env_vars(): - for n in script_env: - print('{} = {}'.format(n, script_env[n])) - -def print_status(msg=''): - print('-- '+msg) - -def get_homebrew_formula(): - tmp_dir = tempfile.mkdtemp() - git(['clone', '--depth', '1', HOMEBREW_REPO_URL, tmp_dir], _out=sys.stdout, _err=sys.stdout) - formula_path = os.path.join(tmp_dir, AZURE_CLI_FORMULA_PATH) - formula_content = [] - with open(formula_path) as f: - formula_content = f.readlines() - formula_content = [x.rstrip() for x in formula_content] - return formula_path, formula_content - -def modify_url(formula_content): - for line_no, line_contents in enumerate(formula_content): - # Find/replace first url - if 'url' in line_contents: - formula_content[line_no] = ' url "' + UPSTREAM_URL_TMPL.format(cli_version=script_env.get('CLI_VERSION')) + '"' - break - -def modify_sha256(formula_content): - for line_no, line_contents in enumerate(formula_content): - # Find/replace first sha256 - if 'sha256' in line_contents: - formula_content[line_no] = ' sha256 "' + script_env.get('CLI_DOWNLOAD_SHA256') + '"' - break - -def _should_include_resource(r): - return not r.startswith('azure-cli') and r not in ['futures'] - -def modify_resources(formula_content): - start_resources_line_no = None - end_resources_line_no = None - for line_no, line_contents in enumerate(formula_content): - if 'resource' in line_contents and start_resources_line_no is None: - start_resources_line_no = line_no - if 'def install' in line_contents and end_resources_line_no is None: - end_resources_line_no = line_no - 1 - break - # Delete resources block - del formula_content[start_resources_line_no : end_resources_line_no] - # The script will have installed homebrew-pypi-poet by this point so we can import - from poet.poet import make_graph, RESOURCE_TEMPLATE - nodes = make_graph('azure-cli') - filtered_nodes = OrderedDict([(n, nodes[n]) for n in nodes if _should_include_resource(n)]) - resources_stanza = '\n\n'.join([RESOURCE_TEMPLATE.render(resource=node) for node in filtered_nodes.values()]) - formula_content[start_resources_line_no:start_resources_line_no] = resources_stanza.split('\n') - - -def write_file(formula_path, formula_content): - with open(formula_path, 'w') as f: - for line in formula_content: - f.write("%s\n" % line) - -def setup_pip_deps(): - pip(['install', '--ignore-installed', 'azure-cli', 'homebrew-pypi-poet'], _out=sys.stdout, _err=sys.stdout) - -def main(): - print_env_vars() - formula_path, formula_content = get_homebrew_formula() - setup_pip_deps() - modify_url(formula_content) - modify_sha256(formula_content) - modify_resources(formula_content) - write_file(formula_path, formula_content) - print_status('Done') - print_status('The new Homebrew formula is available at {}'.format(formula_path)) - print_status('Create a PR to {} for {}'.format(HOMEBREW_REPO_URL, AZURE_CLI_FORMULA_PATH)) - - -if __name__ == '__main__': - main() diff --git a/scripts/releaser/release-rpm.py b/scripts/releaser/release-rpm.py deleted file mode 100644 index 4a9774d6cbd..00000000000 --- a/scripts/releaser/release-rpm.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# This script is interactive as you need to log in to 'az'. - -from __future__ import print_function - -import os -import sys -import time -from datetime import datetime -from six import StringIO -from sh import az, ssh - - -script_env = {} - -def add_script_env(name): - script_env[name] = os.environ.get(name) - -add_script_env('REPO_NAME') -add_script_env('CLI_VERSION') -add_script_env('CLI_DOWNLOAD_SHA256') -add_script_env('AZURE_STORAGE_CONNECTION_STRING') -add_script_env('YUM_REPO_ID') -add_script_env('MS_REPO_URL') -add_script_env('MS_REPO_USERNAME') -add_script_env('MS_REPO_PASSWORD') - -assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set. {}".format(script_env) - -REPO_UPLOAD_SCRIPT_TMPL = """ -import os, requests -payload = {{'name': 'azure-cli', 'version': '{cli_version}', 'repositoryId': '{repo_id}', 'sourceUrl': '{source_url}'}} -r = requests.post('{repo_package_url}', verify=False, auth=('{repo_user}', '{repo_pass}'), json=payload) -print('Status Code') -print(r.status_code) -print('Query with a GET to the following:') -print(r.headers['Location']) -""" - -def print_env_vars(): - for n in script_env: - print('{} = {}'.format(n, script_env[n])) - - -def print_status(msg=''): - print('-- '+msg) - -def print_heading(heading): - print('{0}\n{1}\n{0}'.format('=' * len(heading), heading)) - -def give_chance_to_cancel(msg_prefix=''): - cancel_time_secs = 10 - msg_tmpl = '{}: Starting in {} seconds.' - for i in range(cancel_time_secs, 0, -1): - print_status(msg_tmpl.format(msg_prefix, i)) - time.sleep(1) - -def main(): - print_env_vars() - time_str = datetime.utcnow().strftime('%Y%m%d%H%M%S') - az(["login"], _out=sys.stdout, _err=sys.stdout) - resource_group = 'azurecli-release-rpm-' + time_str - vm_name = 'vm-rpm-' + time_str - print_status('Creating resource group.') - az(['group', 'create', '-l', 'westus', '-n', resource_group], _out=sys.stdout, _err=sys.stdout) - print_status('Creating VM.') - az(['vm', 'create', '-g', resource_group, '-n', vm_name, '--generate-ssh-keys', '--authentication-type', 'ssh', - '--image', 'OpenLogic:CentOS:7.3:latest', '--admin-username', 'myuser'], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - print_status('Getting VM IP address.') - az(['vm', 'list-ip-addresses', '--resource-group', resource_group, '--name', vm_name, - '--query', '[0].virtualMachine.network.publicIpAddresses[0].ipAddress'], _out=io) - ip_address = io.getvalue().strip().replace('"', '') - print_status('VM IP address is {}'.format(ip_address)) - io.close() - vm_connect_str = "myuser@{}".format(ip_address) - my_vm = ssh.bake(['-oStrictHostKeyChecking=no', vm_connect_str]) - print_status('Installing git.') - build_prereqs = "sudo yum update -y && sudo yum install -y git gcc rpm-build rpm-devel rpmlint make python bash coreutils " \ - "diffutils patch rpmdevtools python libffi-devel python-devel openssl-devel" - my_vm(build_prereqs.split(), - _out=sys.stdout, _err=sys.stdout) - my_vm("mkdir -p ~/rpmbuild/BUILD ~/rpmbuild/RPMS ~/rpmbuild/SOURCES ~/rpmbuild/SPECS ~/rpmbuild/SRPMS".split(), _out=io) - io = StringIO() - my_vm(['mktemp', '-d'], _out=io) - repo_dir = io.getvalue().strip() - io.close() - print_status('Cloning repo.') - my_vm(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_dir], _out=sys.stdout, _err=sys.stdout) - path_to_spec_file = os.path.join(repo_dir, 'packaged_releases', 'rpm', 'azure-cli.spec') - print_status('Running build script.') - my_vm(['export', 'CLI_VERSION={}'.format(script_env.get('CLI_VERSION')), '&&', - 'export', 'CLI_DOWNLOAD_SHA256={}'.format(script_env.get('CLI_DOWNLOAD_SHA256')), '&&', - 'rpmbuild', '-v', '-bb', '--clean', path_to_spec_file], - _out=sys.stdout, _err=sys.stdout) - print_status('Build complete.') - io = StringIO() - my_vm(['ls', '~/rpmbuild/RPMS/*/*'], _out=io) - rpm_file_path = io.getvalue().strip() - io.close() - artifact_name = rpm_file_path.split('/')[-1] - print_status('Installing the .rpm on the build machine') - my_vm(['sudo', 'rpm', '-i', rpm_file_path], _out=sys.stdout, _err=sys.stdout) - # Upload to Azure Storage - print_status('Uploading .rpm to Azure storage.') - my_vm(['az', 'storage', 'container', 'create', '--name', 'rpms', '--public-access', 'blob', - '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], - _out=sys.stdout, _err=sys.stdout) - my_vm(['az', 'storage', 'blob', 'upload', '-f', rpm_file_path, - '-n', artifact_name, '-c', 'rpms', '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], - _out=sys.stdout, _err=sys.stdout) - io = StringIO() - my_vm(['az', 'storage', 'blob', 'url', '-n', artifact_name, '-c', 'rpms', '--output', 'tsv', - '--connection-string', '"{}"'.format(script_env.get('AZURE_STORAGE_CONNECTION_STRING'))], _out=io) - rpm_url = io.getvalue().strip() - io.close() - print_status('RPM file uploaded to the following URL.') - print_status(rpm_url) - # Publish to service - my_vm(['wget', '-q', 'https://bootstrap.pypa.io/get-pip.py'], _out=sys.stdout, _err=sys.stdout) - my_vm(['sudo', 'python', 'get-pip.py'], _out=sys.stdout, _err=sys.stdout) - my_vm(['sudo', 'pip', 'install', '--upgrade', 'requests'], _out=sys.stdout, _err=sys.stdout) - upload_script = REPO_UPLOAD_SCRIPT_TMPL.format(cli_version=script_env.get('CLI_VERSION'), - repo_id=script_env.get('YUM_REPO_ID'), - source_url=rpm_url, - repo_package_url=script_env.get('MS_REPO_URL'), - repo_user=script_env.get('MS_REPO_USERNAME'), - repo_pass=script_env.get('MS_REPO_PASSWORD')) - my_vm(['echo', '-e', '"{}"'.format(upload_script), '>>', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) - # Keeping this code commented for when we can automate the signing of RPM packages. - # my_vm(['python', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) - print_status('PRINTING OUT REPO UPLOAD SCRIPT AS THE UNSIGNED RPM NEEDS TO BE FIRST SIGNED BEFORE UPLOADING...') - my_vm(['cat', 'repo_upload.py'], _out=sys.stdout, _err=sys.stdout) - print_status('Done. :)') - give_chance_to_cancel('Delete resource group (in background)') - az(['group', 'delete', '--name', resource_group, '--yes', '--no-wait'], _out=sys.stdout, _err=sys.stdout) - print_status('Finished. :)') - -if __name__ == '__main__': - main() diff --git a/scripts/releaser/release.py b/scripts/releaser/release.py deleted file mode 100644 index 48e1c17d55f..00000000000 --- a/scripts/releaser/release.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# pylint: disable=line-too-long - -from __future__ import print_function, unicode_literals - -import os -import sys -import tempfile -import glob -import re -import time -import fileinput -import requests -import hashlib - -from datetime import datetime - -from subprocess import check_call, check_output, CalledProcessError -from uritemplate import URITemplate, expand - - -script_env = {} - -def add_script_env(name): - script_env[name] = os.environ.get(name) - -add_script_env('REPO_NAME') -add_script_env('GITHUB_USER') -add_script_env('GITHUB_USER_TOKEN') -add_script_env('PYPI_REPO') -# although not used directly here, twine env vars are needed for releasing -add_script_env('TWINE_USERNAME') -add_script_env('TWINE_PASSWORD') -# the new version of the CLI -add_script_env('CLI_VERSION') -add_script_env('AZURE_STORAGE_CONNECTION_STRING') - -assert (all(script_env[n] != None for n in script_env)), "Not all required environment variables have been set. {}".format(script_env) - -GITHUB_API_AUTH = (script_env.get('GITHUB_USER'), script_env.get('GITHUB_USER_TOKEN')) -GITHUB_API_HEADERS = {'Accept': 'application/vnd.github.v3+json', 'user-agent': 'azure-cli-pypi-github-releaser/v1'} - -SOURCE_ARCHIVE_NAME = 'source.tar.gz' - -GITHUB_RELEASE_BODY_TMPL = """ -The module has been published to PyPI. - -View HISTORY.rst of the module for a changelog. - -{} - -Full release notes at https://docs.microsoft.com/en-us/cli/azure/release-notes-azure-cli - -""" - -COMMAND_MODULE_PREFIX = 'azure-cli-' -MODULES_TO_ALWAYS_RELEASE = ['azure-cli'] -MODULES_TO_SKIP = ['azure-cli-testsdk'] - -def give_chance_to_cancel(msg_prefix=''): - cancel_time_secs = 10 - msg_tmpl = '{}: Starting in {} seconds.' - for i in range(cancel_time_secs, 0, -1): - print_status(msg_tmpl.format(msg_prefix, i)) - time.sleep(1) - -def print_env_vars(): - for n in script_env: - print('{} = {}'.format(n, script_env[n])) - -def print_status(msg=''): - print('-- '+msg) - -def print_heading(heading): - print('{0}\n{1}\n{0}'.format('=' * len(heading), heading)) - -def _get_core_modules_paths(repo_working_dir): - for path in glob.glob(repo_working_dir + '/src/*/setup.py'): - yield os.path.basename(os.path.dirname(path)), os.path.dirname(path) - -def _get_command_modules_paths(repo_working_dir, include_prefix=False): - for path in glob.glob(repo_working_dir + '/src/command_modules/{}*/setup.py'.format( - COMMAND_MODULE_PREFIX)): - folder = os.path.dirname(path) - name = os.path.basename(folder) - if not include_prefix: - name = name[len(COMMAND_MODULE_PREFIX):] - yield name, folder - -def _get_all_module_paths(repo_working_dir): - return list(_get_core_modules_paths(repo_working_dir)) + list(_get_command_modules_paths(repo_working_dir, include_prefix=True)) - -def _get_current_module_version(mod_path): - mod_version = None - with open(os.path.join(mod_path, 'setup.py'), 'r') as fh: - version_re = re.compile('VERSION = *') - lines = fh.readlines() - for _, line in enumerate(lines): - if version_re.match(line): - mod_version = line.split('=')[1].strip(' "\'').split('+')[0] - return mod_version - -def clone_repo(repo_working_dir): - check_call(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), repo_working_dir]) - check_call(['git', 'checkout', 'master'], cwd=repo_working_dir) - -def should_release_module(mod_name, mod_path, repo_working_dir): - if mod_name in MODULES_TO_ALWAYS_RELEASE: - print_status('We always release {}.'.format(mod_name)) - return True - if mod_name in MODULES_TO_SKIP: - print_status('Skipping module {} as in modules to skip list.'.format(mod_name)) - return False - # Determine if should release based on the current version - cur_mod_version = _get_current_module_version(mod_path) - r_start = '{}-{}'.format(mod_name, cur_mod_version) - revision_range = "{}..{}".format(r_start, 'HEAD') - try: - module_changes = check_output(["git", "log", "--pretty=format:* %s", revision_range, "--", mod_path, ":(exclude)*/tests/*"], - cwd=repo_working_dir) - except CalledProcessError: - # Maybe the revision_range is invalid if this is a new module. - return True - if module_changes: - print_status('Begin changes in {}'.format(mod_name)) - print(str(module_changes, 'utf-8')) - print_status('End changes in {}'.format(mod_name)) - return True - print_status('Skipping module {} as there are no changes.'.format(mod_name)) - return False - -def modify_setuppy_version(mod_name, mod_path): - setuppy_path = os.path.join(mod_path, 'setup.py') - with open(setuppy_path, 'r') as fh: - version_re = re.compile('VERSION = *') - lines = fh.readlines() - for index, line in enumerate(lines): - if version_re.match(line): - old_version = line.split('=')[1].strip(' "\'').split('+')[0] - major, minor, rev = old_version.split('.') - rev = int(rev) + 1 - version = '{}.{}.{}'.format(major, minor, rev) - lines[index] = 'VERSION = "{}+dev"\n'.format(version) - update_setup = lines - break - else: - raise ValueError('In the setup file {}, version is not found.'.format(setuppy_path)) - if update_setup: - with open(setuppy_path, 'w') as fh: - fh.writelines(update_setup) - else: - raise ValueError('No updated content for setup.py in {}.'.format(mod_name)) - return old_version, version - -def modify_initpy_version(mod_name, mod_path, old_version, new_version): - if mod_name == 'azure-cli': - path_to_init = os.path.join(mod_path, 'azure', 'cli', '__init__.py') - elif mod_name == 'azure-cli-core': - path_to_init = os.path.join(mod_path, 'azure', 'cli', 'core', '__init__.py') - for _, line in enumerate(fileinput.input(path_to_init, inplace=1)): - if line.startswith('__version__'): - sys.stdout.write(line.replace(old_version, new_version)) - else: - sys.stdout.write(line) - -def modify_historyrst(mod_name, mod_path, old_version, new_version): - historyrst_path = os.path.join(mod_path, 'HISTORY.rst') - new_history_lines = [] - just_seen_unreleased = False - contains_unreleased = False - with open(historyrst_path, 'r') as fq: - lines = fq.readlines() - for _, line in enumerate(lines): - if 'unreleased' in line.lower() and not line.startswith('* '): - contains_unreleased = True - if contains_unreleased: - for _, line in enumerate(lines): - if just_seen_unreleased: - # skip the line as it's just a heading for the old unreleased section - just_seen_unreleased = False - continue - if 'unreleased' in line.lower() and not line.startswith('* '): - new_heading = '{} ({})'.format(new_version, datetime.utcnow().strftime('%Y-%m-%d')) - line = '{}\n{}\n'.format(new_heading, '+' * len(new_heading)) - just_seen_unreleased = True - new_history_lines.append(line) - else: - for index, line in enumerate(lines): - if line.startswith('Release History'): - begin = index + 2 - if old_version in line: - end = index - break - new_heading = '{} ({})'.format(new_version, datetime.utcnow().strftime('%Y-%m-%d')) - line = '{}\n{}\n'.format(new_heading, '+' * len(new_heading)) - release_notes = [line] - if mod_name in MODULES_TO_ALWAYS_RELEASE: - release_notes.append('* no changes\n\n') - else: - release_notes.append('* minor fixes\n\n') - new_history_lines = lines[:begin] + release_notes + lines[end:] - with open(historyrst_path, 'w') as fq: - fq.writelines(new_history_lines) - - -def release_module(mod_name, mod_path, repo_working_dir): - # Change version in setup.py - old_version, new_version = modify_setuppy_version(mod_name, mod_path) - # Need to modify __init__.py for these modules as well - if mod_name in ['azure-cli', 'azure-cli-core']: - modify_initpy_version(mod_name, mod_path, old_version, new_version) - # Modify HISTORY.rst - modify_historyrst(mod_name, mod_path, old_version, new_version) - # Create commit with appropriate message. - commit_message = 'Release {} {}'.format(mod_name, new_version) - check_call(['git', 'commit', '-am', commit_message], cwd=repo_working_dir) - commitish = check_output(['git', 'rev-parse', 'HEAD'], cwd=repo_working_dir) - commitish = str(commitish, 'utf-8') - commitish = commitish.strip() - return mod_name, commitish, new_version - - -def install_cli_into_venv(): - venv_dir = tempfile.mkdtemp() - check_call(['virtualenv', venv_dir]) - path_to_pip = os.path.join(venv_dir, 'bin', 'pip') - extra_index_url = 'https://testpypi.python.org/simple' if script_env.get('PYPI_REPO') == 'https://test.pypi.org/legacy/' else None - args = [path_to_pip, 'install', 'azure-cli'] - if extra_index_url: - args.extend(['--extra-index-url', extra_index_url]) - check_call(args) - deps = check_output([path_to_pip, 'freeze']) - deps = str(deps, 'utf-8') - deps = deps.split('\n') - cli_components = [] - for dep in deps: - if dep.startswith('azure-cli'): - cli_components.append(dep.split('==')) - return cli_components - -def run_push_to_git(): - repo_working_dir = tempfile.mkdtemp() - clone_repo(repo_working_dir) - configure_git(repo_working_dir) - commitish_list = [] - for mod_name, mod_path in _get_all_module_paths(repo_working_dir): - print_heading(mod_name.upper()) - if should_release_module(mod_name, mod_path, repo_working_dir): - mod_name, commitish, new_version = release_module(mod_name, mod_path, repo_working_dir) - commitish_list.append((mod_name, commitish, new_version)) - else: - print_status('Skipped {}'.format(mod_name)) - # Push all commits to master. - check_call(['git', 'push', '-f', 'origin', 'master'], cwd=repo_working_dir) - return commitish_list - -def set_up_cli_repo_dir(): - working_dir = tempfile.mkdtemp() - check_call(['git', 'clone', 'https://github.com/{}'.format(script_env.get('REPO_NAME')), working_dir]) - check_call(['pip', 'install', '-e', 'tools'], cwd=working_dir) - return working_dir - -def publish_to_pypi(working_dir, commitish_list): - # Publish all in commitish list to PyPI - assets_dir_map = {} - for mod_name, commitish, _ in commitish_list: - assets_dir = tempfile.mkdtemp() - check_call(['git', 'checkout', commitish], cwd=working_dir) - check_call(['python', '-m', 'tools.automation.release.run', '-c', mod_name, - '-r', script_env.get('PYPI_REPO'), '--dest', assets_dir], cwd=working_dir) - assets_dir_map[mod_name] = assets_dir - # reset back - check_call(['git', 'checkout', 'master'], cwd=working_dir) - return assets_dir_map - -def upload_asset(upload_uri_tmpl, filepath, label): - filename = os.path.basename(filepath) - upload_url = URITemplate(upload_uri_tmpl).expand(name=filename, label=label) - headers = GITHUB_API_HEADERS - headers['Content-Type'] = 'application/octet-stream' - with open(filepath, 'rb') as payload: - requests.post(upload_url, data=payload, auth=GITHUB_API_AUTH, headers=headers) - -def upload_assets_for_github_release(upload_uri_tmpl, component_name, component_version, assets_dir): - for filename in os.listdir(assets_dir): - fullpath = os.path.join(assets_dir, filename) - if filename == SOURCE_ARCHIVE_NAME: - upload_asset(upload_uri_tmpl, fullpath, '{} {} source code (.tar.gz)'.format(component_name, component_version)) - elif filename.endswith('.tar.gz'): - upload_asset(upload_uri_tmpl, fullpath, '{} {} Source Distribution (.tar.gz)'.format(component_name, component_version)) - elif filename.endswith('.whl'): - upload_asset(upload_uri_tmpl, fullpath, '{} {} Python Wheel (.whl)'.format(component_name, component_version)) - -def run_create_github_release(commitish_list, assets_dir_map): - # Create Github release (inc. the artifacts .whl etc.). - print_heading('Creating GitHub releases') - for mod_name, commitish, mod_version in commitish_list: - print_status('Publishing GitHub release for {} {}'.format(mod_name, mod_version)) - tag_name = '{}-{}'.format(mod_name, mod_version) - release_name = "{} {}".format(mod_name, mod_version) - if script_env.get('PYPI_REPO') == 'https://upload.pypi.org/legacy/': - released_pypi_url = 'https://pypi.org/project/{}/{}'.format(mod_name, mod_version) - elif script_env.get('PYPI_REPO') == 'https://test.pypi.org/legacy/': - released_pypi_url = 'https://test.pypi.org/project/{}/{}'.format(mod_name, mod_version) - else: - released_pypi_url = '' - payload = {'tag_name': tag_name, "target_commitish": commitish, "name": release_name, "body": GITHUB_RELEASE_BODY_TMPL.format(released_pypi_url), "prerelease": False} - r = requests.post('https://api.github.com/repos/{}/releases'.format(script_env.get('REPO_NAME')), json=payload, auth=GITHUB_API_AUTH, headers=GITHUB_API_HEADERS) - if r.status_code == 201: - upload_url = r.json()['upload_url'] - upload_assets_for_github_release(upload_url, mod_name, mod_version, assets_dir_map[mod_name]) - print_status('Published GitHub release for {} {}'.format(mod_name, mod_version)) - else: - print_status('ERROR: Failed to create GitHub release for {} {}'.format(mod_name, mod_version)) - -def run_create_packaged_release(working_dir): - # After releasing, create a new venv, and pip install and verify then create - # list of components for the package release step. - print_status('Start installing CLI into venv') - components_list = install_cli_into_venv() - print_status('Finished installing CLI into venv') - archive_dir = tempfile.mkdtemp() - # create the packaged releases automatically - args = ['python', '-m', 'tools.automation.release.packaged', '--version', script_env.get('CLI_VERSION'), '--dest', archive_dir, '--components'] - for name, version in components_list: - # The tag for this module is slightly different so make that change. - if name == 'azure-cli-command-modules-nspkg': - name = 'azure-cli-command_modules-nspkg' - args.append('{}={}'.format(name, version)) - print_status(' '.join(args)) - check_call(args, cwd=working_dir) - print_status('Created packaged release in dir {}'.format(archive_dir)) - # Get the sha256sum - archive_file_name = os.listdir(archive_dir)[0] - archive_file_path = os.path.join(archive_dir, archive_file_name) - sha256 = hashlib.sha256() - with open(archive_file_path, 'rb') as f: - sha256.update(f.read()) - computed_hash = sha256.hexdigest() - print_status('SHA256 of {} is {}'.format(archive_file_path, computed_hash)) - # Upload release archive to Azure Storage - check_call(['az', 'storage', 'blob', 'upload', '--file', archive_file_path, '--name', archive_file_name, '--container-name', 'releases', '--connection-string', script_env.get('AZURE_STORAGE_CONNECTION_STRING')]) - archive_url = check_output(['az', 'storage', 'blob', 'url', '--name', archive_file_name, '--container-name', 'releases', '--connection-string', script_env.get('AZURE_STORAGE_CONNECTION_STRING'), '--output', 'tsv']) - archive_url = str(archive_url, 'utf-8') - archive_url = archive_url.strip() - print_status('Archive URL is {}'.format(archive_url)) - -def configure_git(repo_working_dir): - check_call(['git', 'config', 'user.email', '{}@users.noreply.github.com'.format(script_env.get('GITHUB_USER'))], cwd=repo_working_dir) - check_call(['git', 'config', 'user.name', script_env.get('GITHUB_USER')], cwd=repo_working_dir) - check_call(['git', 'remote', 'set-url', 'origin', 'https://{}:{}@github.com/{}'.format(script_env.get('GITHUB_USER'), script_env.get('GITHUB_USER_TOKEN'), script_env.get('REPO_NAME'))], cwd=repo_working_dir) - -if __name__ == "__main__": - print_env_vars() - give_chance_to_cancel('Create Git release commits') - release_commitish_list = run_push_to_git() - cli_repo_dir = set_up_cli_repo_dir() - give_chance_to_cancel('Publish to PyPI') - release_assets_dir_map = publish_to_pypi(cli_repo_dir, release_commitish_list) - give_chance_to_cancel('Create GitHub releases and tags') - run_create_github_release(release_commitish_list, release_assets_dir_map) - give_chance_to_cancel('Create Packaged Release archive') - # We need to clone the repo again as we've now pushed the git tags and we need them to create the packaged release. - # (we could do 'git pull' but this is easier and uses a clean directory just to be safe) - cli_repo_dir = set_up_cli_repo_dir() - run_create_packaged_release(cli_repo_dir) - print_status('Done.') diff --git a/tools/automation/release/README.md b/tools/automation/release/README.md deleted file mode 100644 index fbe8ee27569..00000000000 --- a/tools/automation/release/README.md +++ /dev/null @@ -1,56 +0,0 @@ -Examples of use -=============== -NOTE: -Wheel and Twine are required. -Use the command below to install them if not installed. -pip install --upgrade wheel twine - -Show help ---------- -``` -$ python -m automation.release.run -h -``` - -Build azure-cli-core --------------------- -``` -$ python -m automation.release.run -c azure-cli-core -``` - -Build azure-cli-core without patching the version number --------------------------------------------------------- -``` -$ python -m automation.release.run -c azure-cli-core --no-version-patch -``` - -Build & Publish azure-cli-core to test PyPI -------------------------------------------- -``` -$ export TWINE_USERNAME= -$ export TWINE_PASSWORD= -$ python -m automation.release.run -c azure-cli-core -r https://testpypi.python.org/pypi -``` - -Build & Publish azure-cli-core to public PyPI ---------------------------------------------- -``` -$ export TWINE_USERNAME= -$ export TWINE_PASSWORD= -$ python -m automation.release.run -c azure-cli-core -r https://pypi.python.org/pypi -``` - - -Examples of checking for component changes since git tag -======================================================== - -List changes for all components since all-v0.1.0b11 ---------------------------------------------------- -``` -$ python -m automation.release.check -s all-v0.1.0b11 -``` - -List changes for azure-cli-core since all-v0.1.0b11 ---------------------------------------------------- -``` -$ python -m automation.release.check -c azure-cli-core -s all-v0.1.0b11 -``` diff --git a/tools/automation/release/__init__.py b/tools/automation/release/__init__.py deleted file mode 100644 index 57e5dc19594..00000000000 --- a/tools/automation/release/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -"""Component/Package release automation code""" diff --git a/tools/automation/release/check.py b/tools/automation/release/check.py deleted file mode 100644 index 6748cb964f7..00000000000 --- a/tools/automation/release/check.py +++ /dev/null @@ -1,71 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import sys -import re -import argparse -from subprocess import check_call, CalledProcessError - -from ..utilities.path import get_repo_root, get_all_module_paths - -REGEX_COMPONENT_NAME = re.compile(r"([a-z\-]*)-([0-9])") - -def error_exit(msg): - print('ERROR: '+msg, file=sys.stderr) - sys.exit(1) - -def check_component_revisions(component_name, r_start, r_end, git_log_verbose=False): - for comp_name, comp_path in get_all_module_paths(): - if comp_name == component_name: - revision_range = "{}..{}".format(r_start, r_end) - try: - log_format = '%C(yellow)%h %Cred%ad %Cblue%an%Cgreen%d %Creset%s' \ - if git_log_verbose else '* %s' - check_call(["git", "log", - "--pretty=format:{}".format(log_format), - revision_range, "--", comp_path, ":(exclude)*/tests/*"], - cwd=get_repo_root()) - except CalledProcessError as e: - error_exit(str(e)) - return - raise error_exit("No component found with name '{}'".format(component_name)) - - -def check_all_component_revisions(r_start, r_end, git_log_verbose=False): - for comp_name, _ in get_all_module_paths(): - print('<<< {} >>>'.format(comp_name)) - check_component_revisions(comp_name, r_start, r_end, git_log_verbose) - print() - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Check for changes made to a component since a git commit or tag. Empty " - "response means no changes.") - parser.add_argument('--component', '-c', - help='Component name (e.g. azure-cli, azure-cli-vm, etc.). If not ' - 'specified and --git-revision-start doesn\'t start with the component ' - 'name, all component changes are shown.') - parser.add_argument('--git-revision-start', '-s', required=True, - help="Git tag (or commit) to use as the start of the revision range. " - "(e.g. release-azure-cli-vm-0.1.0)") - parser.add_argument('--git-revision-end', '-e', default='HEAD', - help='Git tag (or commit) to use as the end of the revision range.') - parser.add_argument('--git-log-verbose', action='store_true', - help='Full git log format with timestamp, author etc.') - args = parser.parse_args() - if args.git_revision_start.startswith('azure-cli') and not args.component: - args.component = re.match(REGEX_COMPONENT_NAME, args.git_revision_start).group(1) - if args.component: - check_component_revisions(args.component, - args.git_revision_start, - args.git_revision_end, - args.git_log_verbose) - else: - check_all_component_revisions(args.git_revision_start, - args.git_revision_end, - args.git_log_verbose) diff --git a/tools/automation/release/notes.py b/tools/automation/release/notes.py deleted file mode 100644 index fc4584fb3cb..00000000000 --- a/tools/automation/release/notes.py +++ /dev/null @@ -1,85 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import os -import sys -import argparse -from datetime import datetime - -from ..utilities.path import get_all_module_paths - -HISTORY_FILENAME = 'HISTORY.rst' - -IGNORE_LINES = ['.. :changelog:', 'Release History'] -UNRELEASED = 'unreleased' - - -def _parse_date_on_line(line): - try: - # Line ends in format '(YYYY-MM-DD)' (ignoring ending whitespace if any) - return _parse_date(line.strip()[-11:-1]) - except (ValueError, TypeError): - return None - - -def get_note_content(history, date_after): - note_content_lines = [] - found_first_release = False - with open(history) as f: - lines = f.read().splitlines() - for line in lines: - if not line or line in IGNORE_LINES or all([char == line[0] for char in line]): - # line empty OR line in ignore list OR line all the same char (likely a heading) - continue - line_date = _parse_date_on_line(line) - if line_date: - found_first_release = True - if line_date and line_date <= date_after: - # Reached the date specified so don't continue - break - if line_date: - # Don't include the date lines in the release notes - continue - if not found_first_release: - # use this to ignore any unreleased notes - continue - # okay to add the line - note_content_lines.append(line) - return '\n'.join(note_content_lines) - - -def generate_release_notes(date_after): - notes = [] - for comp_name, comp_path in get_all_module_paths(): - history_path = os.path.join(comp_path, HISTORY_FILENAME) - if os.path.exists(history_path): - note_content = get_note_content(history_path, date_after) - if note_content: - # Only add note if there where actually changes. - notes.append({'title': comp_name.replace('azure-cli-', '').title(), 'content': note_content}) - return notes - - -def _parse_date(value): - yyyy, mm, dd = value.split('-') - return datetime(int(yyyy), int(mm), int(dd)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Generate release notes for all components. " - "Example 'python -m scripts.automation.release.notes -d 2017-04-03 > release_notes.txt'") - parser.add_argument('--date', '-d', - help='The date after which to collate release notes for (format is YYYY-MM-DD)', - required=True, type=_parse_date) - args = parser.parse_args() - all_notes = generate_release_notes(args.date) - for n in all_notes: - print('### {}'.format(n['title'])) - print() - print(n['content']) - print() diff --git a/tools/automation/release/packaged.py b/tools/automation/release/packaged.py deleted file mode 100644 index a3c1d9d70f1..00000000000 --- a/tools/automation/release/packaged.py +++ /dev/null @@ -1,134 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import argparse -import os -import sys -import tempfile -import tarfile -import shutil -import json -from subprocess import check_call, check_output - -from .version_patcher import VersionPatcher -from ..utilities.path import get_all_module_paths, get_repo_root - -REPO_ROOT_DIR = get_repo_root() -COMPLETION_FILE = os.path.join(REPO_ROOT_DIR, 'packaged_releases', 'az.completion') -ARCHIVE_FILE_TMPL = 'azure-cli_packaged_{}' - - -class Patch(object): # pylint: disable=too-few-public-methods - def __init__(self, src_of_patch, path_to_patch): - """ - - src: Relative path from the repo root - - dest: Relative path to file to patch in the packaged release - """ - self.src_of_patch = src_of_patch - self.path_to_patch = path_to_patch - - def apply(self, working_dir): - src = os.path.join(REPO_ROOT_DIR, self.src_of_patch) - dest = os.path.join(working_dir, self.path_to_patch) - shutil.copy(src, dest) - - -PATCHES = [ - ] - - -def error_exit(msg): - print('ERROR: '+msg, file=sys.stderr) - sys.exit(1) - - -def _gen_tag(c_name, c_version): - return '{}-{}'.format(c_name, c_version) - - -def _verified_tags(components): - available_tags = check_output(['git', 'tag'], cwd=REPO_ROOT_DIR) - available_tags = str(available_tags, 'utf-8') - available_tags = available_tags.split() - for c_name, c_version in components: - t = _gen_tag(c_name, c_version) - if t not in available_tags: - print('Tag {} not found.'.format(t)) - return False - return True - - -def create_packaged_archive(version, components, archive_dest=None, use_version_patch=True): - # Verify the components and versions by checking git tags - if not _verified_tags(components): - error_exit('Some components or versions are not valid.') - working_dir = tempfile.mkdtemp() - print('Using tmp directory {}'.format(working_dir)) - modules = {n: p for n, p in get_all_module_paths()} - cur_git_commitish = check_output(['git', 'rev-parse', 'HEAD'], cwd=REPO_ROOT_DIR).strip() - for c_name, c_version in components: - c_path = modules[c_name] - git_tag = _gen_tag(c_name, c_version) - check_call(['git', 'checkout', git_tag], cwd=REPO_ROOT_DIR) - patcher = VersionPatcher(use_version_patch, c_name, c_path) - patcher.patch() - sub_dir = 'command_modules' if '/command_modules/' in c_path else '' - shutil.copytree(c_path, os.path.join(working_dir, 'src', sub_dir, c_name)) - patcher.unpatch() - check_call(['git', 'checkout', cur_git_commitish], cwd=REPO_ROOT_DIR) - # Add completion file - completion_dest = os.path.join(working_dir, 'az.completion') - shutil.copy(COMPLETION_FILE, completion_dest) - # Apply patches - for patch in PATCHES: - patch.apply(working_dir) - # Build archive - archive_filename = ARCHIVE_FILE_TMPL.format(version) - archive_dest = os.path.expanduser(archive_dest) if archive_dest else os.getcwd() - archive_path = os.path.join(archive_dest, archive_filename+'.tar.gz') - with tarfile.open(archive_path, 'w:gz') as tar: - tar.add(working_dir, arcname=archive_filename) - print("Archive saved to {}".format(archive_path)) - print("Done.") - - -def _type_components_list(value): - c_name, c_version = value.split('=', 1) - if not c_name.startswith('azure-cli'): - c_name = 'azure-cli-' + c_name - return (c_name, c_version) - - -def _type_json_file(value): - with open(os.path.expanduser(value)) as open_file: - data = json.load(open_file) - return [(k, data[k]) for k in data] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Automated generation of the packaged release archive.") - - group = parser.add_mutually_exclusive_group(required=True) - - group.add_argument('--components', '-c', nargs='+', - help="Space separated list in 'component=version' format. " - "(e.g. azure-cli=2.0.0 vm=2.0.0)", - type=_type_components_list) - group.add_argument('--file-data', '-f', - help='Path to JSON file with commands in key/value format. ' - '(e.g. {"azure-cli":"2.0.0", ...})', - type=_type_json_file) - parser.add_argument('--version', '-v', required=True, - help="The version to name the packaged release.") - parser.add_argument('--dest', '-d', - help="The destination directory to place the archive. " - "Defaults to current directory.") - args = parser.parse_args() - - components_list = args.components or args.file_data - create_packaged_archive(args.version, components_list, args.dest) diff --git a/tools/automation/release/run.py b/tools/automation/release/run.py deleted file mode 100644 index 204e5a18011..00000000000 --- a/tools/automation/release/run.py +++ /dev/null @@ -1,85 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from __future__ import print_function - -import argparse -import os -import tempfile -import tarfile -from subprocess import check_call - -from .version_patcher import VersionPatcher -from ..utilities.path import get_all_module_paths - -SOURCE_ARCHIVE_NAME = 'source.tar.gz' - - -def save_source_in_archive(pkg_path, dest): - with tarfile.open(os.path.join(dest, SOURCE_ARCHIVE_NAME), 'w:gz') as tar: - tar.add(pkg_path, arcname=os.path.basename(pkg_path)) - - -def build(pkg_path, dest): - """ - pkg_path - Full path to directory of the package to build - dest - Destination for the built package - """ - check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest, 'sdist', '-d', dest], cwd=pkg_path) - - -def release(pkg_dir, repo): - """Release all packages in a directory""" - pkgs = [os.path.join(pkg_dir, f) for f in os.listdir(pkg_dir) if f != SOURCE_ARCHIVE_NAME] - for pkg in pkgs: - check_call(['twine', 'upload', '--repository-url', repo, '--repository', repo, pkg]) - - -def run_build_release(component_name, repo, dest=None, use_version_patch=True): - """ - component_name - The full component name (e.g. azure-cli, azure-cli-core, azure-cli-vm, etc.) - """ - for comp_name, comp_path in get_all_module_paths(): - if comp_name == component_name: - pkg_dir = dest or tempfile.mkdtemp() - patcher = VersionPatcher(use_version_patch, component_name, comp_path) - patcher.patch() - save_source_in_archive(comp_path, pkg_dir) - build(comp_path, pkg_dir) - patcher.unpatch() - print("Built '{}' to '{}'".format(comp_name, pkg_dir)) - if repo: - release(pkg_dir, repo) - return - raise ValueError("No component found with name '{}'".format(component_name)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description="Automated build and release of a component. To only build, don't specify the" - " repo parameter. The environment variables TWINE_USERNAME and TWINE_PASSWORD " - "are required if releasing.") - parser.add_argument('--component', '-c', required=True, - help='Component name (e.g. azure-cli, azure-cli-vm, etc.)') - parser.add_argument('--no-version-patch', action='store_false', - help="By default, we patch the version number of the package to remove " - "'+dev' if it exists.") - parser.add_argument('--repo', '-r', - help='Repository URL for release (e.g. https://pypi.python.org/pypi, ' - 'https://testpypi.python.org/pypi)') - parser.add_argument('--dest', - help='Directory to store assets. By default, a temp directory is used.') - args = parser.parse_args() - if args.repo: - assert os.environ.get('TWINE_USERNAME') and os.environ.get('TWINE_PASSWORD'), \ - "Set TWINE_USERNAME and TWINE_PASSWORD environment variables to authentication with " \ - "PyPI repository." - if args.dest: - args.dest = os.path.abspath(args.dest) - assert os.path.isdir(args.dest), "Directory '{}' does not exist".format(args.dest) - run_build_release(args.component, - args.repo, - args.dest, - args.no_version_patch) diff --git a/tools/automation/release/version_patcher.py b/tools/automation/release/version_patcher.py deleted file mode 100644 index 172018c9c01..00000000000 --- a/tools/automation/release/version_patcher.py +++ /dev/null @@ -1,77 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import fileinput -import os -import sys - - -class VersionPatcher(object): - """Manages patching the version of a package to remove '+dev' from the version.""" - - def __init__(self, use_version_patch, component_name, component_path): - self.use_version_patch = use_version_patch - self.component_name = component_name - self.component_path = component_path - self.setup_py = os.path.join(component_path, 'setup.py') - self.backup_setup_py_version = None - # These two modules also have version defined in the __init__.py file - # These versions have to be kept in sync. - if self.component_name == 'azure-cli': - self.init_py_path = os.path.join(self.component_path, 'azure', 'cli', '__init__.py') - elif self.component_name == 'azure-cli-core': - self.init_py_path = os.path.join(self.component_path, 'azure', 'cli', 'core', - '__init__.py') - else: - self.init_py_path = None - self.backup_init_version = None - - def _patch_setup_py(self): - for _, line in enumerate(fileinput.input(self.setup_py, inplace=1)): - if line.startswith('VERSION'): - self.backup_setup_py_version = line - # apply version patch - sys.stdout.write(line.replace('+dev', '')) - else: - sys.stdout.write(line) - - def _unpatch_setup_py(self): - for _, line in enumerate(fileinput.input(self.setup_py, inplace=1)): - if line.startswith('VERSION'): - # restore original version - sys.stdout.write(self.backup_setup_py_version) - else: - sys.stdout.write(line) - - def _patch_init_py(self): - for _, line in enumerate(fileinput.input(self.init_py_path, inplace=1)): - if line.startswith('__version__'): - self.backup_init_version = line - # apply init version patch - sys.stdout.write(line.replace('+dev', '')) - else: - sys.stdout.write(line) - - def _unpatch_init_py(self): - for _, line in enumerate(fileinput.input(self.init_py_path, inplace=1)): - if line.startswith('__version__'): - # restore original init version - sys.stdout.write(self.backup_init_version) - else: - sys.stdout.write(line) - - def patch(self): - if not self.use_version_patch: - return - self._patch_setup_py() - if self.init_py_path: - self._patch_init_py() - - def unpatch(self): - if not self.use_version_patch: - return - self._unpatch_setup_py() - if self.init_py_path: - self._unpatch_init_py()