From 083bd0483b5869219db51623eed44a9cd711989b Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 26 Oct 2020 15:30:56 -0700 Subject: [PATCH 1/7] Add azure pipeline config files and templates * Create configure.bat so we can use our skeleton for Windows projects Signed-off-by: Jono Yang --- .travis.yml | 4 +- azure-pipelines.yml | 45 ++++++++++++ configure.bat | 160 +++++++++++++++++++++++++++++++++++++++++ etc/ci/azure-linux.yml | 37 ++++++++++ etc/ci/azure-mac.yml | 36 ++++++++++ etc/ci/azure-win.yml | 36 ++++++++++ 6 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 azure-pipelines.yml create mode 100644 configure.bat create mode 100644 etc/ci/azure-linux.yml create mode 100644 etc/ci/azure-mac.yml create mode 100644 etc/ci/azure-win.yml diff --git a/.travis.yml b/.travis.yml index bcc3be8e..7a342dff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # This is a skeleton Travis CI config file that provides a starting point for adding CI # to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. +# will be specific to that language. # # See https://config.travis-ci.com/ for a full list of configuration options. @@ -18,4 +18,4 @@ python: install: ./configure # Scripts to run at script stage -script: bin/pytest +script: tmp/bin/pytest diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..904ac906 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,45 @@ + +################################################################################ +# We use Azure to run the full tests suites on Python 3.6 +# on Windows (32 and 64), macOS and Linux (64 various distro) +################################################################################ + +jobs: + +################################################################################ +# These jobs are using VMs and Azure-provided Python 3.6 +################################################################################ + + - template: etc/ci/azure-linux.yml + parameters: + job_name: vm_ubuntu16_py36 + image_name: ubuntu-16.04 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-mac.yml + parameters: + job_name: macos1015_py36 + image_name: macos-10.15 + python_versions: ['3.6'] + test_suites: + all: bin/py.test -n 2 -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_32_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x86 + test_suites: + all: Scripts\py.test -vvs --reruns=3 + + - template: etc/ci/azure-win.yml + parameters: + job_name: Win2016_64_py36 + image_name: vs2017-win2016 + python_versions: ['3.6'] + python_architecture: x64 + test_suites: + misc: Scripts\py.test -vvs --reruns=3 diff --git a/configure.bat b/configure.bat new file mode 100644 index 00000000..0a2ca00f --- /dev/null +++ b/configure.bat @@ -0,0 +1,160 @@ +@echo OFF +@setlocal +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. + +@rem ################################ +@rem # A configuration script for Windows +@rem # +@rem # The options and (optional) arguments are: +@rem # --clean : this is exclusive of anything else and cleans the environment +@rem # from built and installed files +@rem # +@rem # --python < path to python.exe> : this must be the first argument and set +@rem # the path to the Python executable to use. If < path to python.exe> is +@rem # set to "path", then the executable will be the python.exe available +@rem # in the PATH. +@rem # +@rem # : this must be the last argument and sets the path to a +@rem # configuration directory to use. +@rem ################################ + +@rem ################################ +@rem # Defaults. Change these variables to customize this script locally +@rem ################################ +@rem # you can define one or more thirdparty dirs, each where the varibale name +@rem # is prefixed with TPP_DIR +set "TPP_DIR=thirdparty" + +@rem # default configurations for dev +set "CONF_DEFAULT=etc/conf/dev" + +@rem # default thirdparty dist for dev +if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( + set "TPP_DIR_DEV=thirdparty/dev" +) + +@rem # default supported version for Python 3 +set SUPPORTED_PYTHON3=3.6 + +@rem ################################# + +@rem python --version +@rem python -c "import sys;print(sys.executable)" + + +@rem Current directory where this .bat files lives +set CFG_ROOT_DIR=%~dp0 + +@rem path where a configured Python should live in the current virtualenv if installed +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe + +set PYTHON_EXECUTABLE= + +@rem parse command line options and arguments +:collectopts +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b +if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts + +@rem We are not cleaning: Either we have a provided configure config path or we use a default. +if ""%1""=="""" ( + set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% +) else ( + set CFG_CMD_LINE_ARGS=%1 +) + +@rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run +if exist ""%CONFIGURED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% + goto run +) + +@rem If we have a command arg for Python use this as-is +if ""%PROVIDED_PYTHON%""==""path"" ( + @rem use a bare python available in the PATH + set PYTHON_EXECUTABLE=python + goto run +) +if exist ""%PROVIDED_PYTHON%"" ( + set PYTHON_EXECUTABLE=%PROVIDED_PYTHON% + goto run +) + + +@rem otherwise we search for a suitable Python interpreter +:find_python + +@rem First check the existence of the "py" launcher (available in Python 3) +@rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 +@rem if not, check if we have an old py 2.7 +@rem exist if all fails + +where py >nul 2>nul +if %ERRORLEVEL% == 0 ( + @rem we have a py launcher, check for the availability of our required Python 3 version + py -3.6 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -3.6 + ) else ( + @rem we have no required python 3, let's try python 2: + py -2 --version >nul 2>nul + if %ERRORLEVEL% == 0 ( + set PYTHON_EXECUTABLE=py -2 + ) else ( + @rem we have py and no python 3 and 2, exit + echo * Unable to find an installation of Python. + exit /b 1 + ) + ) +) else ( + @rem we have no py launcher, check for a default Python 2 installation + if not exist ""%DEFAULT_PYTHON2%"" ( + echo * Unable to find an installation of Python. + exit /b 1 + ) else ( + set PYTHON_EXECUTABLE=%DEFAULT_PYTHON2% + ) +) + +:run + +@rem without this things may not always work on Windows 10, but this makes things slower +set PYTHONDONTWRITEBYTECODE=1 + +call mkdir "%CFG_ROOT_DIR%tmp" +call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtualenv.pyz +call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" +call "%CFG_ROOT_DIR%tmp\Scripts\activate" +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel + + +@rem Return a proper return code on failure +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) +endlocal +goto activate + + +:cli_help +echo A configuration script for Windows +echo usage: configure [options] [path/to/config/directory] +echo. +echo The options and arguments are: +echo [path/to/config/directory] : this optionally sets the path to a +echo configuration directory to use. Defaults to etc/conf/dev if not set +echo. +echo --clean : this is exclusive of anything else and cleans the environment +echo from built and installed files +echo. +echo --python path/to/python.exe : this is set to the path of an alternative +echo Python executable to use. If path/to/python.exe is set to "path", +echo then the executable will be the python.exe available in the PATH. +echo. + + +:activate +@rem Activate the virtualenv +if exist "%CFG_ROOT_DIR%Scripts\activate" ( + "%CFG_ROOT_DIR%Scripts\activate" +) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml new file mode 100644 index 00000000..2e12e5b0 --- /dev/null +++ b/etc/ci/azure-linux.yml @@ -0,0 +1,37 @@ +parameters: + job_name: '' + image_name: 'ubuntu-16.04' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-mac.yml new file mode 100644 index 00000000..752ae2e6 --- /dev/null +++ b/etc/ci/azure-mac.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: ./configure + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml new file mode 100644 index 00000000..62208574 --- /dev/null +++ b/etc/ci/azure-win.yml @@ -0,0 +1,36 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x86 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each pyver in parameters.python_versions }}: + ${{ each tsuite in parameters.test_suites }}: + ${{ format('py{0} {1}', pyver, tsuite.key) }}: + python_version: ${{ pyver }} + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + steps: + - checkout: self + fetchDepth: 10 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + architecture: '${{ parameters.python_architecture }}' + displayName: 'Install Python $(python_version)' + + - script: configure --python path + displayName: 'Run Configure' + + - script: $(test_suite) + displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' From 847564ead159cfcebe0f04066daa6f23e1a5a123 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:40:38 -0700 Subject: [PATCH 2/7] Fix path to Scripts directory * Remove unused variables and options Signed-off-by: Jono Yang --- configure.bat | 48 ++++-------------------------------------------- 1 file changed, 4 insertions(+), 44 deletions(-) diff --git a/configure.bat b/configure.bat index 0a2ca00f..cbb42446 100644 --- a/configure.bat +++ b/configure.bat @@ -13,41 +13,12 @@ @rem # the path to the Python executable to use. If < path to python.exe> is @rem # set to "path", then the executable will be the python.exe available @rem # in the PATH. -@rem # -@rem # : this must be the last argument and sets the path to a -@rem # configuration directory to use. -@rem ################################ - @rem ################################ -@rem # Defaults. Change these variables to customize this script locally -@rem ################################ -@rem # you can define one or more thirdparty dirs, each where the varibale name -@rem # is prefixed with TPP_DIR -set "TPP_DIR=thirdparty" - -@rem # default configurations for dev -set "CONF_DEFAULT=etc/conf/dev" - -@rem # default thirdparty dist for dev -if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( - set "TPP_DIR_DEV=thirdparty/dev" -) - -@rem # default supported version for Python 3 -set SUPPORTED_PYTHON3=3.6 - -@rem ################################# - -@rem python --version -@rem python -c "import sys;print(sys.executable)" - @rem Current directory where this .bat files lives set CFG_ROOT_DIR=%~dp0 - @rem path where a configured Python should live in the current virtualenv if installed -set CONFIGURED_PYTHON=%CFG_ROOT_DIR%Scripts\python.exe - +set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= @rem parse command line options and arguments @@ -56,13 +27,6 @@ if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (call rmdir /s /q "%CFG_ROOT_DIR%tmp") && call exit /b if "%1" EQU "--python" (set PROVIDED_PYTHON=%~2) && shift && shift && goto collectopts -@rem We are not cleaning: Either we have a provided configure config path or we use a default. -if ""%1""=="""" ( - set CFG_CMD_LINE_ARGS=%CONF_DEFAULT% -) else ( - set CFG_CMD_LINE_ARGS=%1 -) - @rem If we have a pre-configured Python in our virtualenv, reuse this as-is and run if exist ""%CONFIGURED_PYTHON%"" ( set PYTHON_EXECUTABLE=%CONFIGURED_PYTHON% @@ -83,7 +47,6 @@ if exist ""%PROVIDED_PYTHON%"" ( @rem otherwise we search for a suitable Python interpreter :find_python - @rem First check the existence of the "py" launcher (available in Python 3) @rem if we have it, check if we have a py -3 installed with the good version or a py 2.7 @rem if not, check if we have an old py 2.7 @@ -116,8 +79,8 @@ if %ERRORLEVEL% == 0 ( ) ) -:run +:run @rem without this things may not always work on Windows 10, but this makes things slower set PYTHONDONTWRITEBYTECODE=1 @@ -141,9 +104,6 @@ echo A configuration script for Windows echo usage: configure [options] [path/to/config/directory] echo. echo The options and arguments are: -echo [path/to/config/directory] : this optionally sets the path to a -echo configuration directory to use. Defaults to etc/conf/dev if not set -echo. echo --clean : this is exclusive of anything else and cleans the environment echo from built and installed files echo. @@ -155,6 +115,6 @@ echo. :activate @rem Activate the virtualenv -if exist "%CFG_ROOT_DIR%Scripts\activate" ( - "%CFG_ROOT_DIR%Scripts\activate" +if exist "%CFG_ROOT_DIR%tmp\Scripts\activate" ( + "%CFG_ROOT_DIR%tmp\Scripts\activate" ) From 0f293cb11374d96df99757ce8dc6bed4730c9751 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 10:45:16 -0700 Subject: [PATCH 3/7] Install current project in configure.bat Signed-off-by: Jono Yang --- configure.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index cbb42446..958f5bf8 100644 --- a/configure.bat +++ b/configure.bat @@ -21,6 +21,7 @@ set CFG_ROOT_DIR=%~dp0 set CONFIGURED_PYTHON=%CFG_ROOT_DIR%tmp\Scripts\python.exe set PYTHON_EXECUTABLE= + @rem parse command line options and arguments :collectopts if "%1" EQU "--help" (goto cli_help) @@ -89,7 +90,7 @@ call curl -o "%CFG_ROOT_DIR%tmp\virtualenv.pyz" https://bootstrap.pypa.io/virtua call %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%tmp\virtualenv.pyz" "%CFG_ROOT_DIR%tmp" call "%CFG_ROOT_DIR%tmp\Scripts\activate" call "%CFG_ROOT_DIR%tmp\Scripts\pip" install --upgrade pip virtualenv setuptools wheel - +call "%CFG_ROOT_DIR%tmp\Scripts\pip" install -e .[testing] @rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( From 63f6946e1b3b070924b156a86b0ed0c3da6b7a48 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 11:41:51 -0700 Subject: [PATCH 4/7] Call pytest from proper location * Remove rerun option from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 904ac906..cf84da29 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,7 +16,7 @@ jobs: image_name: ubuntu-16.04 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-mac.yml parameters: @@ -24,7 +24,7 @@ jobs: image_name: macos-10.15 python_versions: ['3.6'] test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 + all: tmp/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -33,7 +33,7 @@ jobs: python_versions: ['3.6'] python_architecture: x86 test_suites: - all: Scripts\py.test -vvs --reruns=3 + all: tmp\Scripts\pytest -vvs - template: etc/ci/azure-win.yml parameters: @@ -42,4 +42,4 @@ jobs: python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: Scripts\py.test -vvs --reruns=3 + misc: tmp\Scripts\pytest -vvs From 7fd250691c0f9db772e289d3cfb2224d70f3dcd0 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 27 Oct 2020 12:43:10 -0700 Subject: [PATCH 5/7] Use newer VM images on Azure Signed-off-by: Jono Yang --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf84da29..fad6928d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,8 +12,8 @@ jobs: - template: etc/ci/azure-linux.yml parameters: - job_name: vm_ubuntu16_py36 - image_name: ubuntu-16.04 + job_name: ubuntu18_py36 + image_name: ubuntu-18.04 python_versions: ['3.6'] test_suites: all: tmp/bin/pytest -n 2 -vvs @@ -28,8 +28,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_32_py36 - image_name: vs2017-win2016 + job_name: win2019_32_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x86 test_suites: @@ -37,9 +37,9 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: Win2016_64_py36 - image_name: vs2017-win2016 + job_name: win2019_64_py36 + image_name: windows-2019 python_versions: ['3.6'] python_architecture: x64 test_suites: - misc: tmp\Scripts\pytest -vvs + all: tmp\Scripts\pytest -vvs From bceb8f98633bda15982c667848dc86be19ee6f97 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 28 Oct 2020 17:03:27 -0700 Subject: [PATCH 6/7] Add .gitattributes * We have this to ensure the line ending of configure.bat is always CRLF Signed-off-by: Jono Yang --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2d555b20 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Set configure.bat's line ending to CRLF. Sometimes batch scripts don't work +# properly on Windows if the line ending is LF and not CRLF +configure.bat eol=crlf From e772fe670da017a9ff51e3b7119996f12169e79c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 3 Nov 2020 19:32:39 -0800 Subject: [PATCH 7/7] Clean template Signed-off-by: Jono Yang --- etc/ci/azure-linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml index 2e12e5b0..752ae2e6 100644 --- a/etc/ci/azure-linux.yml +++ b/etc/ci/azure-linux.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: '' python_versions: [] test_suites: {} python_architecture: x64 @@ -19,7 +19,6 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} - steps: - checkout: self fetchDepth: 10