From 2ad2f05fad6d6118ba44de387075eadb2bb1c48f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 10 May 2021 16:48:36 +0200 Subject: [PATCH] Update configure scripts Drop using Python etc/configure.py. Instead use a plain bash and plain bat scripts. Improve the way the Python interpreter is picked using either the PYTHON_EXECUTABLE environment variable or the PYTHON_EXECUTABLE file and fallback to use the default Python foudn in the PATH. Update release, other scripts and CI scripts accordingly Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 16 +- MANIFEST.in | 2 + appveyor.yml | 2 +- azure-pipelines.yml | 14 +- configure | 219 ++++++++++---- configure.bat | 236 +++++++++------ etc/ci/azure-linux.yml | 46 --- etc/ci/{azure-mac.yml => azure-posix.yml} | 17 +- etc/ci/azure-win.yml | 10 +- etc/configure.py | 335 ---------------------- etc/release/scancode-create-release.sh | 10 +- extractcode | 2 +- extractcode.bat | 2 +- scancode | 2 +- scancode.bat | 4 +- 15 files changed, 363 insertions(+), 554 deletions(-) delete mode 100644 etc/ci/azure-linux.yml rename etc/ci/{azure-mac.yml => azure-posix.yml} (77%) delete mode 100644 etc/configure.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d80486dac7a..09d0b7548d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Changelog ========= -v21.4.x (next) +v21.x.x (next) -------------- Breaking API changes: @@ -24,6 +24,19 @@ Breaking API changes: multiple manifests for a single package instance. +v21.5.11 +-------- + +Breaking API changes: +~~~~~~~~~~~~~~~~~~~~~ + + - The configure scripts for Linux, macOS and Windows are new. These are now + only native scripts (.bat on Windows and .sh on POSIX) and the Python script + etc/configure.py has been removed. Use the PYTHON_EXECUTABLE enviroment + varibale to point to alternative non-default Python executable. An this on + all OSes. + + Ouputs: ~~~~~~~ @@ -43,6 +56,7 @@ License scanning: - Till Jaeger @LeChasseur + v21.3.31 -------- diff --git a/MANIFEST.in b/MANIFEST.in index 114ec213651..8e2543a707e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,6 +16,8 @@ include *.toml include extractcode* include scancode* +# path to executable used by a built app +include PYTHON_EXECUTABLE include *.rst include Dockerfile diff --git a/appveyor.yml b/appveyor.yml index 9769b2d5f70..dd9de265e34 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,4 +16,4 @@ build: off test_script: - python -c "import sys;print(sys.getdefaultencoding())" - - cmd: "set PYTHON_EXE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode" + - cmd: "set PYTHON_EXECUTABLE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b7b23aaeb2f..d9c93082971 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: # These jobs are using VMs and Azure-provided Pythons 3.6 ################################################################################ - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: vm_ubuntu16_py36 image_name: ubuntu-16.04 @@ -95,7 +95,7 @@ jobs: bin/pytest -n 3 -vvs --test-suite=all \ tests/licensedcode/test_zzzz_cache.py - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos_1014_py3x image_name: macos-10.14 @@ -104,7 +104,7 @@ jobs: all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py - - template: etc/ci/azure-mac.yml + - template: etc/ci/azure-posix.yml parameters: job_name: macos_1015_py3x image_name: macos-10.15 @@ -113,7 +113,7 @@ jobs: all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py -# - template: etc/ci/azure-mac.yml +# - template: etc/ci/azure-posix.yml # parameters: # job_name: macos_110_py3x # image_name: macos-11.0 @@ -142,7 +142,7 @@ jobs: all: Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: vm_ubuntu16_py3x image_name: ubuntu-16.04 @@ -151,7 +151,7 @@ jobs: all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: vm_ubuntu18_py3x image_name: ubuntu-18.04 @@ -160,7 +160,7 @@ jobs: all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py - - template: etc/ci/azure-linux.yml + - template: etc/ci/azure-posix.yml parameters: job_name: vm_ubuntu20_py3x image_name: ubuntu-20.04 diff --git a/configure b/configure index b39d0b2d032..02a69f132de 100755 --- a/configure +++ b/configure @@ -3,100 +3,199 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# ScanCode is a trademark of nexB Inc. -# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://github.com/nexB/ for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + set -e +#set -x + +################################ +# A configuration script to set things up: +# create a virtualenv and install or update thirdparty packages. +# Source this script for initial configuration +# Use configure --help for details +# +# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz +# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +################################ +CLI_ARGS=$1 + +################################ +# Defaults. Change these variables to customize this script +################################ + +# Requirement arguments passed to pip and used by default or with --dev. +REQUIREMENTS="--editable . --constraint requirements.txt" +DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt" + +# where we create a virtualenv +VIRTUALENV_DIR=. + +# Cleanable files and directories to delete with the --clean option +CLEANABLE=" + build + bin + lib + lib64 + include + tcl + local + .Python + .eggs + pip-selfcheck.json + src/scancode_toolkit.egg-info + SCANCODE_DEV_MODE + man + Scripts" + +# extra arguments passed to pip +PIP_EXTRA_ARGS=" " + +# the URL to download virtualenv.pyz if needed +VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +################################ -if [[ "$OS" == "Windows_NT" ]]; then - echo "You seem to be running on Windows under Cygwin / MSYS(2)," - echo "like e.g. Git for Windows Bash. This script does not properly work in" - echo "this scenario. As a Windows user, please run 'configure.bat' from a " - echo "regular command prompt instead." - exit 1 + +################################ +# Current directory where this script lives +CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + + +################################ +# scancode-specific: Thirdparty package locations and index handling +# Do we have thirdparty/files? use the mit.LICENSE as a proxy +if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then + LINKS=$CFG_ROOT_DIR/thirdparty +else + LINKS=https://thirdparty.aboutcode.org/pypi fi +PIP_EXTRA_ARGS="--no-index --find-links $LINKS" +################################ ################################ -# A configuration script to set things up: create a virtualenv and install -# update thirdparty packages +# Set the quiet flag to empty if not defined +if [[ "$CFG_QUIET" == "" ]]; then + CFG_QUIET=" " +fi + + +################################ +# find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi +fi -function cli_help { + +################################ +cli_help() { echo An initial configuration script echo " usage: ./configure [options]" echo - echo The default is to configure for regular use. - echo The script will attempt to find a suitable Python executable. - echo Set the PYTHON_EXECUTABLE environment variable to provide your own - echo Python executable path. + echo The default is to configure for regular use. Use --dev for development. echo echo The options are: - echo "--clean: clean built and installed files and exit." - echo "--dev: configure the environment for development." - echo "--help: display these help messages and exit." + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. set +e exit } -################################ -# Defaults. Change these variables to customize this script locally -################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} -# thirdparty package locations -export THIRDPARTY_DIR="thirdparty" -export THIRDPARTY_LINKS="https://thirdparty.aboutcode.org/pypi" -# requirements used by default or with --dev. -# note the use of constraints with -c -REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt" +create_virtualenv() { + # create a virtualenv for Python + # Note: we do not use the bundled Python 3 "venv" because its behavior and + # presence is not consistent across Linux distro and sometimes pip is not + # included either by default. The virtualenv.pyz app cures all these issues. + + VENV_DIR="$1" + if [ ! -f "$CFG_BIN_DIR/python" ]; then + + mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" + + if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then + VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" + else + VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + fi + + $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ + --wheel embed --pip embed --setuptools embed \ + --seeder pip \ + --never-download \ + --no-periodic-update \ + --no-vcs-ignore \ + $CFG_QUIET \ + "$CFG_ROOT_DIR/$VENV_DIR" + fi +} -# default supported Python version -if [[ "$CONFIGURE_SUPPORTED_PYTHON" == "" ]]; then - CONFIGURE_SUPPORTED_PYTHON=3.6 -fi -################################ +install_packages() { + # install requirements in virtualenv + # note: --no-build-isolation means that pip/wheel/setuptools will not + # be reinstalled a second time and reused from the virtualenv and this + # speeds up the installation. + # We always have the PEP517 build dependencies installed already. + + "$CFG_BIN_DIR/pip" install \ + --upgrade \ + --no-build-isolation \ + $CFG_QUIET \ + $PIP_EXTRA_ARGS \ + $1 +} -# Current directory where this script lives -PROJECT_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +################################ +# Main command line entry point +CFG_DEV_MODE=0 +CFG_REQUIREMENTS=$REQUIREMENTS -# parse command line options -CLI_ARGS="$REQUIREMENTS" -CONFIGURE_DEV_MODE=0 -case "$1" in +case "$CLI_ARGS" in --help) cli_help;; - --clean) CLI_ARGS=--clean;; - --dev) CLI_ARGS="$DEV_REQUIREMENTS" && CONFIGURE_DEV_MODE=1;; + --clean) clean;; + --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; esac - -# find a proper Python to run - -if [[ "$CONFIGURE_PYTHON_EXECUTABLE" == "" ]]; then - CONFIGURE_PYTHON_EXECUTABLE=python3 -fi +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" ################################ +# scancode-specific: Setup development mode -# Setup development mode - -if [[ "$CONFIGURE_DEV_MODE" == 1 ]]; then +if [[ "$CFG_DEV_MODE" == 1 ]]; then # Add development tag file to auto-regen license index on file changes - touch $PROJECT_ROOT_DIR/SCANCODE_DEV_MODE -fi - -# Run configure scripts proper and activate - -$CONFIGURE_PYTHON_EXECUTABLE "$PROJECT_ROOT_DIR/etc/configure.py" $CLI_ARGS - -# Activate the virtualenv if it exists -if [[ -f "$PROJECT_ROOT_DIR/bin/activate" ]]; then - source "$PROJECT_ROOT_DIR/bin/activate" + touch "$CFG_ROOT_DIR/SCANCODE_DEV_MODE" fi set +e diff --git a/configure.bat b/configure.bat index ee69da6ff74..c499247206f 100644 --- a/configure.bat +++ b/configure.bat @@ -4,122 +4,198 @@ @rem Copyright (c) nexB Inc. and others. All rights reserved. @rem SPDX-License-Identifier: Apache-2.0 @rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -@rem ScanCode is a trademark of nexB Inc. -@rem See https://github.com/nexB/scancode-toolkit for support or download. +@rem See https://github.com/nexB/ for support or download. @rem See https://aboutcode.org for more information about nexB OSS projects. + +@rem ################################ +@rem # A configuration script to set things up: +@rem # create a virtualenv and install or update thirdparty packages. +@rem # Source this script for initial configuration +@rem # Use configure --help for details + +@rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz +@rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ -@rem # A configuration script to set things up: create a virtualenv and install -@rem # update thirdparty packages -goto config -:cli_help - echo An initial configuration script - echo " usage: configure [options]" - echo. - echo The default is to configure for regular use. - echo The script will attempt to find a suitable Python executable. - echo Set the PYTHON_EXECUTABLE environment variable to provide your own - echo Python executable path. - echo. - echo The options are: - echo "--clean: clean built and installed files and exit." - echo "--dev: configure the environment for development." - echo "--help: display these help messages and exit." - echo. - endlocal - exit /b 0 -:config @rem ################################ -@rem # Defaults. Change these variables to customize this script locally +@rem # Defaults. Change these variables to customize this script @rem ################################ -@rem # thirdparty package locations -set "THIRDPARTY_DIR=thirdparty" -set "THIRDPARTY_LINKS=https://thirdparty.aboutcode.org/pypi" - -@rem # requirements used by defaults and with --dev +@rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt" -@rem # default supported Python version -if not defined CONFIGURE_SUPPORTED_PYTHON ( - set CONFIGURE_SUPPORTED_PYTHON=3.6 -) +@rem # where we create a virtualenv +set "VIRTUALENV_DIR=." -@rem ################################# +@rem # Cleanable files and directories to delete with the --clean option +set "CLEANABLE=build Scripts Lib include tcl local .Python .eggs pip-selfcheck.json src/scancode_toolkit.egg-info SCANCODE_DEV_MODE man" -@rem Current directory where this script lives -set PROJECT_ROOT_DIR=%~dp0 +@rem # extra arguments passed to pip +set "PIP_EXTRA_ARGS= " +@rem # the URL to download virtualenv.pyz if needed +set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +@rem ################################ -@rem parse command line options -set CLI_ARGS="%REQUIREMENTS%" -set CONFIGURE_DEV_MODE=0 -if "%1" EQU "--help" (goto cli_help) -if "%1" EQU "--clean" (set CLI_ARGS=--clean) -if "%1" EQU "--dev" ( - set CLI_ARGS=%DEV_REQUIREMENTS% - set CONFIGURE_DEV_MODE=1 + +@rem ################################ +@rem # Current directory where this script lives +set CFG_ROOT_DIR=%~dp0 +set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + + +@rem ################################ +@rem # scancode-specific: Thirdparty package locations and index handling +@rem # Do we have thirdparty/files? use the mit.LICENSE as a proxy +if exist ""%CFG_ROOT_DIR%\thirdparty\mit.LICENSE"" ( + set "LINKS=%CFG_ROOT_DIR%\thirdparty" +) else ( + set "LINKS=https://thirdparty.aboutcode.org/pypi" ) +set "PIP_EXTRA_ARGS=--no-index --find-links %LINKS%" +@rem ################################ -@rem find a proper Python to run -if defined CONFIGURE_PYTHON_EXECUTABLE ( - if exist "%CONFIGURE_PYTHON_EXECUTABLE%" ( - goto run - ) +@rem ################################ +@rem # Set the quiet flag to empty if not defined +if not defined CFG_QUIET ( + set "CFG_QUIET= " ) -:find_python -@rem Check the existence of the "py" launcher available in Python 3 -@rem If we have it, check if we have a py -3 installed with the required version -@rem Try to use a Python in the path if all else fail +@rem ################################ +@rem # Main command line entry point +set CFG_DEV_MODE=0 +set "CFG_REQUIREMENTS=%REQUIREMENTS%" + +if "%1" EQU "--help" (goto cli_help) +if "%1" EQU "--clean" (goto clean) +if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + set CFG_DEV_MODE=1 +) +if "%1" EQU "--python" ( + echo "The --python is now DEPRECATED. Use the PYTHON_EXECUTABLE environment + echo "variable instead. Run configure --help for details." + exit /b 0 +) -where py >nul 2>nul -if %ERRORLEVEL% == 0 ( - @rem we have a py launcher, check for the availability of our Python 3 version - set CONFIGURE_PYTHON_EXECUTABLE=py -%CONFIGURE_SUPPORTED_PYTHON%% - %CONFIGURE_PYTHON_EXECUTABLE% --version >nul 2>nul - if %ERRORLEVEL% neq 0 ( - echo "* Unable to find a suitable installation of Python %CONFIGURE_SUPPORTED_PYTHON%." - exit /b 1 +################################ +@rem # find a proper Python to run +@rem # Use environment variables or a file if available. +@rem # Otherwise the latest Python by default. +if not defined PYTHON_EXECUTABLE ( + @rem # check for a file named PYTHON_EXECUTABLE + if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( + set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" + ) else ( + set "PYTHON_EXECUTABLE=py" ) -) else ( - echo "* Unable to find a suitable installation of Python %CONFIGURE_SUPPORTED_PYTHON%." - exit /b 1 ) -:run -@rem ################################ +:create_virtualenv +@rem # create a virtualenv for Python +@rem # Note: we do not use the bundled Python 3 "venv" because its behavior and +@rem # presence is not consistent across Linux distro and sometimes pip is not +@rem # included either by default. The virtualenv.pyz app cures all these issues. -@rem # Setup development mode +if not exist ""%CFG_BIN_DIR%\python.exe"" ( + if not exist "%CFG_BIN_DIR%" ( + mkdir %CFG_BIN_DIR% + ) -if "%CONFIGURE_DEV_MODE%" == 1 ( - @rem # Add development tag file to auto-regen license index on file changes - echo. 2>%%PROJECT_ROOT_DIR%\SCANCODE_DEV_MODE" + if exist ""%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz"" ( + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + ) else ( + if not exist ""%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz"" ( + curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% + + if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% + ) + ) + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ + --wheel embed --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + ) ) -@rem # Run configure scripts proper and activate +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) -@rem without this there are some heisenbugs on Windows 10 -@rem but this make scancode run slower -set PYTHONDONTWRITEBYTECODE=1 +:install_packages +@rem # install requirements in virtualenv +@rem # note: --no-build-isolation means that pip/wheel/setuptools will not +@rem # be reinstalled a second time and reused from the virtualenv and this +@rem # speeds up the installation. +@rem # We always have the PEP517 build dependencies installed already. -call %CONFIGURE_PYTHON_EXECUTABLE% "%PROJECT_ROOT_DIR%etc\configure.py" %CLI_ARGS% +%CFG_BIN_DIR%\pip install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% -@rem Return a proper return code on failure if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% ) -endlocal -@rem # Activate the virtualenv if it exists -if exist "%PROJECT_ROOT_DIR%Scripts\activate" ( - "%PROJECT_ROOT_DIR%Scripts\activate" +@rem ################################ +@rem # scancode-specific: Setup development mode +if "%CFG_DEV_MODE%" == 1 ( + @rem # Add development tag file to auto-regen license index on file changes + echo.>%CFG_ROOT_DIR%\SCANCODE_DEV_MODE +) +@rem ################################ + +exit /b 0 + + +@rem ################################ + +:cli_help + echo An initial configuration script + echo " usage: configure [options]" + echo " " + echo The default is to configure for regular use. Use --dev for development. + echo " " + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo " " + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + exit /b 0 + + +:clean +@rem # Remove cleanable file and directories and files from the root dir. +echo "* Cleaning ..." +for %%F in (%CLEANABLE%) do ( + rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 + del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) exit /b 0 diff --git a/etc/ci/azure-linux.yml b/etc/ci/azure-linux.yml deleted file mode 100644 index f7d314543e5..00000000000 --- a/etc/ci/azure-linux.yml +++ /dev/null @@ -1,46 +0,0 @@ -parameters: - job_name: '' - image_name: 'ubuntu-16.04' - python_versions: [] - test_suites: {} - python_architecture: x64 - -jobs: - - job: ${{ parameters.job_name }} - - pool: - vmImage: ${{ parameters.image_name }} - - strategy: - matrix: - ${{ each pyver in parameters.python_versions }}: - ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} - test_suite_label: ${{ tsuite.key }} - test_suite: ${{ tsuite.value }} - - steps: - - checkout: self - fetchDepth: 10 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' - - - script: python --version - displayName: 'Get python version' - - - script: python3 --version - displayName: 'Get python3 version' - - - script: python$(python_version) --version - displayName: 'Get pythonXXX version' - - - script: ./configure --dev - displayName: 'Run Configure' - - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-mac.yml b/etc/ci/azure-posix.yml similarity index 77% rename from etc/ci/azure-mac.yml rename to etc/ci/azure-posix.yml index bab712ed295..7a9acff4b35 100644 --- a/etc/ci/azure-mac.yml +++ b/etc/ci/azure-posix.yml @@ -19,6 +19,7 @@ jobs: python_version: ${{ pyver }} test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 @@ -29,16 +30,12 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: python --version - displayName: 'Get python version' - - - script: python3 --version - displayName: 'Get python3 version' - - - script: python$(python_version) --version - displayName: 'Get pythonXXX version' - - - script: ./configure --dev + - script: | + python --version + python3 --version + python$(python_version) --version + echo "python$(python_version)" > PYTHON_EXECUTABLE + ./configure --dev displayName: 'Run Configure' - script: $(test_suite) diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 2891049b03e..ad326bae08e 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -3,7 +3,7 @@ parameters: image_name: '' python_versions: [] test_suites: {} - python_architecture: x86 + python_architecture: x64 jobs: - job: ${{ parameters.job_name }} @@ -29,10 +29,10 @@ jobs: architecture: '${{ parameters.python_architecture }}' displayName: 'Install Python $(python_version)' - - script: python --version - displayName: 'Get python version' - - - script: configure --dev + - script: | + python --version + echo | set /p=py $(python_version)> PYTHON_EXECUTABLE + configure --dev displayName: 'Run Configure' - script: $(test_suite) diff --git a/etc/configure.py b/etc/configure.py deleted file mode 100644 index 018776d53fe..00000000000 --- a/etc/configure.py +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -""" -A configuration helper that wraps virtualenv and pip to configure an Python -isolated virtual environment and install requirement there with pip using -bundled packages from a third-party directory. It does some minimal checks on -supported OSes, architectures and Python versions. - -To use, create requirements flies and use this way: - -* to configure and install:: - - ./configure.py [, ...] - -* to cleanup everything:: - - ./configure.py --clean -""" - -import os -import shutil -import subprocess -import sys - - -def unsupported(platform): - print('Unsupported Python, OS, platform or architecture: {platform}'.format(platform=platform)) - print('See https://github.com/nexB/scancode-toolkit/ for supported OS/platforms. ' - 'Enter a ticket https://github.com/nexB/scancode-toolkit/issues ' - 'asking for support of your OS/platform combo.') - sys.exit(1) - -# Supported platforms and arches -# Supported Python versions - - -sys_platform = str(sys.platform).lower() - -if not( - sys_platform.startswith('linux') - or 'win32' in sys_platform - or 'darwin' in sys_platform - or 'freebsd' in sys_platform -): - unsupported(sys_platform) - -if not (sys.maxsize > 2 ** 32): - unsupported('32 bits: use a 64 bits OS and Python instead.') - -if sys.version_info < (3, 6): - unsupported('Only Python 64 bits 3.6 and above are supported.') - -on_win = 'win32' in sys_platform - -if on_win: - BIN_DIR_NAME = 'Scripts' -else: - BIN_DIR_NAME = 'bin' - - -def call(cmd): - """ - Run the `cmd` command (as a list of args) with all env vars using `root_dir` - as the current working directory. - """ - cmd = ' '.join(cmd) - try: - subprocess.check_call( - cmd, - shell=True, - env=dict(os.environ), - cwd=ROOT_DIR, - stderr=subprocess.STDOUT, - ) - except Exception as e: - raise Exception('Failed to run {}\n{}'.format(cmd, str(e))) - - -# list of cleanble directory and file paths -cleanable = ''' - build - bin - lib - lib64 - include - tcl - local - .Python - .eggs - pip-selfcheck.json - src/scancode_toolkit.egg-info - SCANCODE_DEV_MODE - man - Scripts -'''.split() - - -def find_pycache(root_dir): - """ - Yield __pycache__ directory paths found in root_dir as paths relative to - root_dir. - """ - for top, dirs, _files in os.walk(root_dir): - for d in dirs: - if d != '__pycache__': - continue - dir_path = os.path.join(top, d) - dir_path = dir_path.replace(root_dir, '', 1) - dir_path = dir_path.strip(os.path.sep) - yield dir_path - - -def clean(root_dir, cleanable=cleanable): - """ - Remove `cleanable` directories and files from `root_dir`. - """ - print('* Cleaning ...') - - # also clean __pycache__ if any - cleanable.extend(find_pycache(root_dir)) - - for d in cleanable: - loc = os.path.join(root_dir, d) - if os.path.exists(loc): - if os.path.isdir(loc): - shutil.rmtree(loc) - else: - os.remove(loc) - - -def quote(s): - """ - Return a string s enclosed in double quotes. - """ - return '"{}"'.format(s) - - -def create_virtualenv(root_dir, venv_pyz, quiet=False): - """ - Create a virtualenv in the `root_dir` directory. Use the current Python - exe. Use the virtualenv.pyz app in at `venv_pyz`. - If `quiet` is True, information messages are suppressed. - - Note: we do not use the bundled Python 3 "venv" because its behavior and - presence is not consistent across Linux distro and sometimes pip is not - included either by default. The virtualenv.pyz app cures all these issues. - """ - if not quiet: - print('* Configuring Python ...') - - if not venv_pyz or not os.path.exists(venv_pyz): - print('Configuration Error: Unable to find {venv_pyz}... aborting.'.format(venv_pyz=venv_pyz)) - sys.exit(1) - - standard_python = sys.executable - - # once we have a pyz, we do not want to download anything else nor ever update - vcmd = [ - standard_python, quote(venv_pyz), - '--wheel', 'embed', '--pip', 'embed', '--setuptools', 'embed', - '--seeder', 'pip', - '--never-download', - '--no-periodic-update', - ] - - if quiet: - vcmd += ['-qq'] - - # we create the virtualenv in the root_dir - vcmd += [quote(root_dir)] - call(vcmd) - - -def activate_virtualenv(): - """ - Activate the ROOT_DIR virtualenv in the current process. - """ - activate_this = os.path.join(BIN_DIR, 'activate_this.py') - exec(open(activate_this).read(), {'__file__': activate_this}) - - -def pip_install(req_args, quiet=False): - """ - Install a list of `req_args` command line requirement arguments with - pip, using packages found in THIRDPARTY_DIR_OR_LINKS directory or URL. - """ - if not quiet: - print('* Installing packages ...') - - if on_win: - cmd = [CONFIGURED_PYTHON, '-m', 'pip'] - else: - cmd = [quote(os.path.join(BIN_DIR, 'pip'))] - - # note: --no-build-isolation means that pip/wheel/setuptools will not - # be reinstalled a second time and this speeds up the installation. - # We always have the PEP517 build dependencies installed already. - cmd += [ - 'install', '--upgrade', - '--no-build-isolation', - '--no-index', - '--find-links', THIRDPARTY_DIR_OR_LINKS, - ] - if quiet: - cmd += ['-qq'] - - cmd += req_args - call(cmd) - - -def pip_cache_wheels(requirement_files, quiet=False): - """ - Download and cache wheels from a list of `requirement_files` pip requirement - files using packages found in THIRDPARTY_LINKS and save them in the - THIRDPARTY_DIR directory. - """ - if not quiet: - print('* Downloading packages ...') - - for req_file in requirement_files: - if on_win: - cmd = [CONFIGURED_PYTHON, '-m', 'pip'] - else: - cmd = [quote(os.path.join(BIN_DIR, 'pip'))] - - cmd += [ - 'download', '--no-index', - '--find-links', THIRDPARTY_LINKS, - '--dest', THIRDPARTY_DIR, - ] - - if quiet: - cmd += ['-qq'] - - cmd += ['--requirement', req_file] - - call(cmd) - - -def get_pip_req_files(req_args): - rfs = [f for f in req_args if f.startswith('requirements') and f.endswith('.txt')] - return rfs - - -usage = '\nUsage: configure [--clean] []\n' - -if __name__ == '__main__': - - # you must create a CONFIGURE_QUIET env var if you want to run quietly - ################## - quiet = 'CONFIGURE_QUIET' in os.environ - - # define/setup common directories and locations - ################## - current_dir = os.path.abspath(os.path.dirname(__file__)) - ROOT_DIR = os.path.dirname(current_dir) - sys.path.insert(0, ROOT_DIR) - - BIN_DIR = os.path.join(ROOT_DIR, BIN_DIR_NAME) - if on_win: - CONFIGURED_PYTHON = os.path.join(BIN_DIR, 'python.exe') - else: - CONFIGURED_PYTHON = os.path.join(BIN_DIR, 'python') - - # THIRDPARTY_DIR is a cache of wheels - THIRDPARTY_DIR = os.environ.get('THIRDPARTY_DIR', 'thirdparty') - THIRDPARTY_DIR = os.path.join(ROOT_DIR, THIRDPARTY_DIR) - os.makedirs(THIRDPARTY_DIR, exist_ok=True) - - THIRDPARTY_LINKS = os.environ.get('THIRDPARTY_LINKS', 'https://thirdparty.aboutcode.org/pypi') - - # no_cache = 'CONFIGURE_NO_CACHE' in os.environ - # if no_cache: - # THIRDPARTY_DIR_OR_LINKS = THIRDPARTY_DIR - # else: - # if we have at least one wheel in THIRDPARTY_DIR, we assume we are offline - # otherwise we are online and use our remote links for pip operations - has_wheels = any(w.endswith('.whl') for w in os.listdir(THIRDPARTY_DIR)) - THIRDPARTY_DIR_OR_LINKS = THIRDPARTY_DIR if has_wheels else THIRDPARTY_LINKS - - # collect args - ################## - requirement_args = ['--requirement', 'requirements.txt'] - - args = sys.argv[1:] - if args: - arg0 = args[0] - if arg0 == '--clean': - clean(ROOT_DIR) - sys.exit(0) - - # use provided pip args instead of defaults - requirement_args = args - - # Determine where to get dependencies from - ################################# - - # etc/thirdparty must contain virtualenv.pyz - etc_thirdparty = os.path.join(os.path.dirname(__file__), 'thirdparty') - VIRTUALENV_PYZ_APP_LOC = os.path.join(etc_thirdparty, 'virtualenv.pyz') - if not os.path.exists(VIRTUALENV_PYZ_APP_LOC): - print(( - '* FAILED to configure: virtualenv application {VIRTUALENV_PYZ_APP_LOC} not found. ' - 'The current version needs to be saved in etc/thirdparty. ' - 'See https://github.com/pypa/get-virtualenv and ' - 'https://virtualenv.pypa.io/en/latest/installation.html#via-zipapp' - ).format(VIRTUALENV_PYZ_APP_LOC=VIRTUALENV_PYZ_APP_LOC) - ) - sys.exit(1) - - # Configure proper: create and activate virtualenv - ########################### - if not os.path.exists(CONFIGURED_PYTHON): - create_virtualenv(root_dir=ROOT_DIR, venv_pyz=VIRTUALENV_PYZ_APP_LOC, quiet=quiet) - - activate_virtualenv() - - # cache requirements - req_files = get_pip_req_files(requirement_args) - # pip_cache_wheels(requirement_files=req_files, quiet=quiet) - - # ... and installl - pip_install(requirement_args, quiet=quiet) - - if not quiet: - print('* Configuration completed.') - print() diff --git a/etc/release/scancode-create-release.sh b/etc/release/scancode-create-release.sh index df4804f3875..bdfa50a7a42 100755 --- a/etc/release/scancode-create-release.sh +++ b/etc/release/scancode-create-release.sh @@ -188,7 +188,7 @@ function backup_previous_release { } function clean_build { - rm -rf build dist thirdparty + rm -rf build dist thirdparty PYTHON_EXECUTABLE SCANCODE_DEV_MODE } backup_previous_release @@ -249,16 +249,18 @@ function build_app_archive { echo " " echo "## RELEASE: Building archive for Python $PYTHON_APP_VERSION on operating system: $operating_system" + clean_build + mkdir -p thirdparty + if [ "$operating_system" == "windows" ]; then # create a zip only on Windows formats=zip + echo -n "py 3.6">PYTHON_EXECUTABLE else formats=xztar + echo -n "python3.6">PYTHON_EXECUTABLE fi - clean_build - mkdir -p thirdparty - # 1. Collect thirdparty deps only for the subset for this Python/operating_system bin/python etc/release/fetch_requirements.py \ --requirements-file=requirements.txt \ diff --git a/extractcode b/extractcode index aa9100a8184..eaaa343ce2d 100755 --- a/extractcode +++ b/extractcode @@ -112,7 +112,7 @@ SCANCODE_ROOT_DIR="$( cd "$( dirname "${SCANCODE_BIN}" )" && pwd )" SCANCODE_CONFIGURED_PYTHON="$SCANCODE_ROOT_DIR/bin/python" if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then echo "* Configuring ScanCode for first use..." - CONFIGURE_QUIET=1 "$SCANCODE_ROOT_DIR/configure" + CONFIGURE_QUIET=-qq "$SCANCODE_ROOT_DIR/configure" fi "$SCANCODE_ROOT_DIR/bin/extractcode" "$@" diff --git a/extractcode.bat b/extractcode.bat index d27897c434f..54551411a5e 100644 --- a/extractcode.bat +++ b/extractcode.bat @@ -17,7 +17,7 @@ goto scancode :configure echo * Configuring ScanCode for first use... -set CONFIGURE_QUIET=1 +set CONFIGURE_QUIET=-qq call "%SCANCODE_ROOT_DIR%configure" @rem Return a proper return code on failure diff --git a/scancode b/scancode index c269c093dfc..634615bd710 100755 --- a/scancode +++ b/scancode @@ -114,7 +114,7 @@ SCANCODE_CONFIGURED_PYTHON="$SCANCODE_ROOT_DIR/bin/python" if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then set -e echo "* Configuring ScanCode for first use..." - CONFIGURE_QUIET=1 "$SCANCODE_ROOT_DIR/configure" + CONFIGURE_QUIET=-qq "$SCANCODE_ROOT_DIR/configure" set +e fi diff --git a/scancode.bat b/scancode.bat index e64bf916d2a..516ae135d2f 100644 --- a/scancode.bat +++ b/scancode.bat @@ -17,9 +17,9 @@ goto scancode :configure echo * Configuring ScanCode for first use... -echo * WARNING: Native Windows will be deprecated in the future in favor of Windows Subsystem for Linux +echo * WARNING: Native Windows may be deprecated in the future in favor of Windows Subsystem for Linux 2 echo * WARNING: Please visit https://github.com/nexB/scancode-toolkit/issues/2366 for details and to provide feedback -set CONFIGURE_QUIET=1 +set CONFIGURE_QUIET=-qq call "%SCANCODE_ROOT_DIR%configure" @rem Return a proper return code on failure