Update configure scripts

Drop using Python etc/configure.py. Instead use a plain bash and plain bat scripts. Improve the way the Python interpreter is picked using either the PYTHON_EXECUTABLE environment variable or the PYTHON_EXECUTABLE file and fallback to use the default Python foudn in the PATH. Update release, other scripts and CI scripts accordingly Signed-off-by: Philippe Ombredanne <[email protected]>
mapelpapel · May 10, 2021 · 2ad2f05 · 2ad2f05
1 parent bb04420
commit 2ad2f05
Show file tree

Hide file tree

Showing 15 changed files with 363 additions and 554 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,7 +1,7 @@
 Changelog
 =========
 
-v21.4.x (next)
+v21.x.x (next)
 --------------
 
 Breaking API changes:
@@ -24,6 +24,19 @@ Breaking API changes:
    multiple manifests for a single package instance.
 
 
+v21.5.11
+--------
+
+Breaking API changes:
+~~~~~~~~~~~~~~~~~~~~~
+
+ - The configure scripts for Linux, macOS and Windows are new. These are now
+   only native scripts (.bat on Windows and .sh on POSIX) and the Python script
+   etc/configure.py has been removed. Use the PYTHON_EXECUTABLE enviroment
+   varibale to point to alternative non-default Python executable. An this on
+   all OSes.
+
+
 Ouputs:
 ~~~~~~~
 
@@ -43,6 +56,7 @@ License scanning:
     - Till Jaeger @LeChasseur 
 
 
+
 v21.3.31
 --------
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -16,6 +16,8 @@ include *.toml
 include extractcode*
 include scancode*
 
+# path to executable used by a built app
+include PYTHON_EXECUTABLE
 
 include *.rst
 include Dockerfile

diff --git a/appveyor.yml b/appveyor.yml
@@ -16,4 +16,4 @@ build: off
 
 test_script:
     - python -c "import sys;print(sys.getdefaultencoding())"
-    - cmd: "set PYTHON_EXE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode"
+    - cmd: "set PYTHON_EXECUTABLE=%PYTHON%\\python.exe && configure --dev && Scripts\\pytest -vvs --reruns 2 tests\\scancode"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -10,7 +10,7 @@ jobs:
 # These jobs are using VMs and Azure-provided Pythons 3.6
 ################################################################################
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: vm_ubuntu16_py36
           image_name: ubuntu-16.04
@@ -95,7 +95,7 @@ jobs:
                 bin/pytest -n 3 -vvs --test-suite=all \
                   tests/licensedcode/test_zzzz_cache.py
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos_1014_py3x
           image_name: macos-10.14
@@ -104,7 +104,7 @@ jobs:
               all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
 
-    - template: etc/ci/azure-mac.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: macos_1015_py3x
           image_name: macos-10.15
@@ -113,7 +113,7 @@ jobs:
               all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
 
-#    - template: etc/ci/azure-mac.yml
+#    - template: etc/ci/azure-posix.yml
 #      parameters:
 #          job_name: macos_110_py3x
 #          image_name: macos-11.0
@@ -142,7 +142,7 @@ jobs:
               all: Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py
 
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: vm_ubuntu16_py3x
           image_name: ubuntu-16.04
@@ -151,7 +151,7 @@ jobs:
               all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: vm_ubuntu18_py3x
           image_name: ubuntu-18.04
@@ -160,7 +160,7 @@ jobs:
               all: bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
 
-    - template: etc/ci/azure-linux.yml
+    - template: etc/ci/azure-posix.yml
       parameters:
           job_name: vm_ubuntu20_py3x
           image_name: ubuntu-20.04

diff --git a/configure b/configure
@@ -3,100 +3,199 @@
 # Copyright (c) nexB Inc. and others. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# ScanCode is a trademark of nexB Inc.
-# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://github.com/nexB/ for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+
 set -e
+#set -x
+
+################################
+# A configuration script to set things up: 
+# create a virtualenv and install or update thirdparty packages.
+# Source this script for initial configuration
+# Use configure --help for details
+#
+# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz
+# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default
+################################
+CLI_ARGS=$1
+
+################################
+# Defaults. Change these variables to customize this script
+################################
+
+# Requirement arguments passed to pip and used by default or with --dev.
+REQUIREMENTS="--editable . --constraint requirements.txt"
+DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt"
+
+# where we create a virtualenv
+VIRTUALENV_DIR=.
+
+# Cleanable files and directories to delete with the --clean option
+CLEANABLE="
+    build
+    bin
+    lib
+    lib64
+    include
+    tcl
+    local
+    .Python
+    .eggs
+    pip-selfcheck.json
+    src/scancode_toolkit.egg-info
+    SCANCODE_DEV_MODE
+    man
+    Scripts"
+
+# extra  arguments passed to pip
+PIP_EXTRA_ARGS=" "
+
+# the URL to download virtualenv.pyz if needed
+VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz
+################################
 
-if [[ "$OS" == "Windows_NT" ]]; then
-    echo "You seem to be running on Windows under Cygwin / MSYS(2),"
-    echo "like e.g. Git for Windows Bash. This script does not properly work in"
-    echo "this scenario. As a Windows user, please run 'configure.bat' from a "
-    echo "regular command prompt instead."
-    exit 1
+
+################################
+# Current directory where this script lives
+CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
+
+
+################################
+# scancode-specific: Thirdparty package locations and index handling
+# Do we have thirdparty/files? use the mit.LICENSE as a proxy
+if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
+    LINKS=$CFG_ROOT_DIR/thirdparty
+else
+    LINKS=https://thirdparty.aboutcode.org/pypi
 fi
+PIP_EXTRA_ARGS="--no-index --find-links $LINKS"
+################################
 
 
 ################################
-# A configuration script to set things up: create a virtualenv and install
-# update thirdparty packages
+# Set the quiet flag to empty if not defined 
+if [[ "$CFG_QUIET" == "" ]]; then
+    CFG_QUIET=" "
+fi
+
+
+################################
+# find a proper Python to run
+# Use environment variables or a file if available.
+# Otherwise the latest Python by default.
+if [[ "$PYTHON_EXECUTABLE" == "" ]]; then
+    # check for a file named PYTHON_EXECUTABLE 
+    if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then
+        PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE")
+    else
+        PYTHON_EXECUTABLE=python3
+    fi
+fi
 
-function cli_help {
+
+################################
+cli_help() {
     echo An initial configuration script
     echo "  usage: ./configure [options]"
     echo
-    echo The default is to configure for regular use.
-    echo The script will attempt to find a suitable Python executable.
-    echo Set the PYTHON_EXECUTABLE environment variable to provide your own
-    echo Python executable path.
+    echo The default is to configure for regular use. Use --dev for development.
     echo
     echo The options are:
-    echo  "--clean: clean built and installed files and exit."
-    echo  "--dev:   configure the environment for development."
-    echo  "--help:  display these help messages and exit."
+    echo " --clean: clean built and installed files and exit."
+    echo " --dev:   configure the environment for development."
+    echo " --help:  display this help message and exit."
     echo
+    echo By default, the python interpreter version found in the path is used.
+    echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to
+    echo configure another Python executable interpreter to use. If this is not
+    echo set, a file named PYTHON_EXECUTABLE containing a single line with the
+    echo path of the Python executable to use will be checked last.
     set +e
     exit
 }
 
 
-################################
-# Defaults. Change these variables to customize this script locally
-################################
+clean() {
+    # Remove cleanable file and directories and files from the root dir.
+    echo "* Cleaning ..."
+    for cln in $CLEANABLE;
+        do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}";
+    done
+    set +e
+    exit
+}
 
-#  thirdparty package locations
-export THIRDPARTY_DIR="thirdparty"
-export THIRDPARTY_LINKS="https://thirdparty.aboutcode.org/pypi"
 
-# requirements used by default or with --dev.
-# note the use of constraints with -c 
-REQUIREMENTS="--editable . --constraint requirements.txt"
-DEV_REQUIREMENTS="--editable .[dev] --editable .[packages] --constraint requirements.txt --constraint requirements-dev.txt"
+create_virtualenv() {
+    # create a virtualenv for Python
+    # Note: we do not use the bundled Python 3 "venv" because its behavior and
+    # presence is not consistent across Linux distro and sometimes pip is not
+    # included either by default. The virtualenv.pyz app cures all these issues.
+
+    VENV_DIR="$1"
+    if [ ! -f "$CFG_BIN_DIR/python" ]; then
+
+        mkdir -p "$CFG_ROOT_DIR/$VENV_DIR"
+
+        if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz"
+        else
+            VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz"
+            wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL"
+        fi
+
+        $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \
+            --wheel embed --pip embed --setuptools embed \
+            --seeder pip \
+            --never-download \
+            --no-periodic-update \
+            --no-vcs-ignore \
+            $CFG_QUIET \
+            "$CFG_ROOT_DIR/$VENV_DIR"
+    fi
+}
 
-# default supported Python version
-if [[ "$CONFIGURE_SUPPORTED_PYTHON" == "" ]]; then
-    CONFIGURE_SUPPORTED_PYTHON=3.6
-fi
 
-################################
+install_packages() {
+    # install requirements in virtualenv
+    # note: --no-build-isolation means that pip/wheel/setuptools will not
+    # be reinstalled a second time and reused from the virtualenv and this
+    # speeds up the installation.
+    # We always have the PEP517 build dependencies installed already.
+
+    "$CFG_BIN_DIR/pip" install \
+        --upgrade \
+        --no-build-isolation \
+        $CFG_QUIET \
+        $PIP_EXTRA_ARGS \
+        $1
+}
 
-# Current directory where this script lives
-PROJECT_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+################################
+# Main command line entry point
+CFG_DEV_MODE=0
+CFG_REQUIREMENTS=$REQUIREMENTS
 
-# parse command line options
-CLI_ARGS="$REQUIREMENTS"
-CONFIGURE_DEV_MODE=0
-case "$1" in
+case "$CLI_ARGS" in
     --help)    cli_help;;
-    --clean)   CLI_ARGS=--clean;;
-    --dev)     CLI_ARGS="$DEV_REQUIREMENTS" && CONFIGURE_DEV_MODE=1;;
+    --clean)   clean;;
+    --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
 esac
 
-
-# find a proper Python to run
-
-if [[ "$CONFIGURE_PYTHON_EXECUTABLE" == "" ]]; then
-    CONFIGURE_PYTHON_EXECUTABLE=python3
-fi
+create_virtualenv "$VIRTUALENV_DIR"
+install_packages "$CFG_REQUIREMENTS"
+. "$CFG_BIN_DIR/activate"
 
 ################################
+# scancode-specific: Setup development mode
 
-# Setup development mode
-
-if [[ "$CONFIGURE_DEV_MODE" == 1 ]]; then
+if [[ "$CFG_DEV_MODE" == 1 ]]; then
     # Add development tag file to auto-regen license index on file changes
-    touch $PROJECT_ROOT_DIR/SCANCODE_DEV_MODE
-fi
-
-# Run configure scripts proper and activate
-
-$CONFIGURE_PYTHON_EXECUTABLE "$PROJECT_ROOT_DIR/etc/configure.py" $CLI_ARGS
-
-# Activate the virtualenv if it exists
-if [[ -f "$PROJECT_ROOT_DIR/bin/activate" ]]; then
-    source "$PROJECT_ROOT_DIR/bin/activate"
+    touch "$CFG_ROOT_DIR/SCANCODE_DEV_MODE"
 fi
 
 set +e